You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/10/13 16:11:26 UTC
(superset) branch master updated: refactor(db_engine_specs): dedupe boolean-coercion logic for Databricks/Spark (#45202)
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 4da3ab9277b refactor(db_engine_specs): dedupe boolean-coercion logic for Databricks/Spark (#45202)
4da3ab9277b is described below
commit 4da3ab9277b7fbd23c254e2621c37051b6bb305f
Author: Evan Rusackas <ev...@preset.io>
AuthorDate: Tue Oct 13 09:11:19 2026 -0700
refactor(db_engine_specs): dedupe boolean-coercion logic for Databricks/Spark (#45202)
Co-authored-by: Claude Sonnet 5 <no...@anthropic.com>
---
superset/db_engine_specs/databricks.py | 41 ++++------------------------------
superset/db_engine_specs/hive.py | 18 ++++++++++++++-
superset/db_engine_specs/spark.py | 20 ++---------------
3 files changed, 23 insertions(+), 56 deletions(-)
diff --git a/superset/db_engine_specs/databricks.py b/superset/db_engine_specs/databricks.py
index 1fa207e8ba4..6c0f9211c5f 100644
--- a/superset/db_engine_specs/databricks.py
+++ b/superset/db_engine_specs/databricks.py
@@ -28,7 +28,7 @@ from flask import g
from flask_babel import gettext as __
from marshmallow import fields, Schema
from marshmallow.validate import Range
-from sqlalchemy import false, text, true, types
+from sqlalchemy import text, types
from sqlalchemy.engine.default import DefaultDialect
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
@@ -40,7 +40,7 @@ from superset.db_engine_specs.base import (
BasicParametersMixin,
DatabaseCategory,
)
-from superset.db_engine_specs.hive import HiveEngineSpec
+from superset.db_engine_specs.hive import HiveEngineSpec, SparkSqlBooleanEqualityMixin
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import OAuth2Error
from superset.utils import json
@@ -250,7 +250,7 @@ time_grain_expressions: dict[str | None, str] = {
}
-class DatabricksBaseEngineSpec(BaseEngineSpec):
+class DatabricksBaseEngineSpec(SparkSqlBooleanEqualityMixin, BaseEngineSpec):
"""Base engine specification for Databricks flavors and connection methods."""
_time_grain_expressions = time_grain_expressions
@@ -260,27 +260,6 @@ class DatabricksBaseEngineSpec(BaseEngineSpec):
identifier_quote_start: str = "`"
identifier_quote_end: str = "`"
- # Databricks SQL applies strict type checking: comparing a boolean column
- # against an integer literal (or a bound Python boolean that the
- # Hive/Databricks driver renders as 0/1) fails with
- # DATATYPE_MISMATCH.DATA_DIFF_TYPES (see #36765). Enabling equality
- # operators ensures boolean filters compile as 'col = true' / 'col = false'.
- use_equality_for_boolean_filters: bool = True
-
- @classmethod
- def coerce_boolean_for_sql(cls, value: bool | None) -> Any:
- """
- Render Python booleans as explicit boolean keywords.
-
- The Hive/Databricks SQLAlchemy dialects render Python booleans as
- integer literals (0/1) in some binding paths, so emit sqlalchemy
- true()/false() instead: they always compile to the TRUE/FALSE keywords
- and never reach the driver as bind parameters (#36765).
- """
- if value is None:
- return None
- return true() if value else false()
-
@classmethod
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None
@@ -1167,7 +1146,7 @@ class DatabricksODBCEngineSpec(DatabricksBaseEngineSpec):
# ambiguous-driver fallback in `get_engine_spec` picks the first backend-matching
# spec by definition order, and this spec's inherited `HiveEngineSpec.execute()`
# crashes on the `databricks.sql.client.Cursor` used by unrecognized-driver URIs.
-class DatabricksHiveEngineSpec(HiveEngineSpec):
+class DatabricksHiveEngineSpec(SparkSqlBooleanEqualityMixin, HiveEngineSpec):
"""Databricks engine spec using Hive connector for Interactive Clusters."""
engine_name = "Databricks Interactive Cluster"
@@ -1205,18 +1184,6 @@ class DatabricksHiveEngineSpec(HiveEngineSpec):
# consolidates all Databricks connection methods. This spec exists for
# backwards compatibility with Interactive Cluster connections.
- # Databricks Interactive Clusters apply the same strict type checking as
- # Databricks SQL, so boolean filters need equality against boolean keywords.
- use_equality_for_boolean_filters: bool = True
-
- @classmethod
- def coerce_boolean_for_sql(cls, value: bool | None) -> Any:
- # Same rationale as DatabricksBaseEngineSpec: the Hive driver renders
- # Python booleans as 0/1 in some binding paths (#36765).
- if value is None:
- return None
- return true() if value else false()
-
_show_functions_column = "function"
_time_grain_expressions = time_grain_expressions
diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py
index be93d0de3d7..938f377ab56 100644
--- a/superset/db_engine_specs/hive.py
+++ b/superset/db_engine_specs/hive.py
@@ -30,7 +30,7 @@ import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
from flask import current_app as app, g
-from sqlalchemy import Column, text, types
+from sqlalchemy import Column, false, text, true, types
from sqlalchemy.engine.interfaces import Dialect
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
@@ -85,6 +85,22 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str:
return location
+class SparkSqlBooleanEqualityMixin:
+ """
+ Shared boolean-filter coercion for the Databricks and Spark SQL specs,
+ whose dialects render bound Python booleans as integer literals in some
+ binding paths and apply strict type checking to the result (#36765).
+ """
+
+ use_equality_for_boolean_filters: bool = True
+
+ @classmethod
+ def coerce_boolean_for_sql(cls, value: bool | None) -> Any:
+ if value is None:
+ return None
+ return true() if value else false()
+
+
class HiveEngineSpec(PrestoEngineSpec):
"""Reuses PrestoEngineSpec functionality."""
diff --git a/superset/db_engine_specs/spark.py b/superset/db_engine_specs/spark.py
index 1fa46b359c5..a81473c9238 100644
--- a/superset/db_engine_specs/spark.py
+++ b/superset/db_engine_specs/spark.py
@@ -16,14 +16,11 @@
# under the License.
from __future__ import annotations
-from typing import Any
-
-from sqlalchemy import false, true
from sqlalchemy.dialects import registry
from superset.constants import TimeGrain
from superset.db_engine_specs.base import DatabaseCategory
-from superset.db_engine_specs.hive import HiveEngineSpec
+from superset.db_engine_specs.hive import HiveEngineSpec, SparkSqlBooleanEqualityMixin
time_grain_expressions: dict[str | None, str] = {
None: "{col}",
@@ -44,7 +41,7 @@ time_grain_expressions: dict[str | None, str] = {
}
-class SparkEngineSpec(HiveEngineSpec):
+class SparkEngineSpec(SparkSqlBooleanEqualityMixin, HiveEngineSpec):
"""Engine specification for Apache Spark SQL."""
engine = "spark"
@@ -52,19 +49,6 @@ class SparkEngineSpec(HiveEngineSpec):
_time_grain_expressions = time_grain_expressions
engine_name = "Apache Spark SQL"
- # Spark SQL shares syntax rules with Databricks SQL; equality comparisons
- # are required for boolean filters to avoid type mismatch errors (see #36765).
- use_equality_for_boolean_filters: bool = True
-
- @classmethod
- def coerce_boolean_for_sql(cls, value: bool | None) -> Any:
- # The pyhive driver renders Python booleans as 0/1 in some binding
- # paths, and Spark SQL applies the same strict type checking as
- # Databricks SQL (#36765).
- if value is None:
- return None
- return true() if value else false()
-
metadata = {
"description": "Apache Spark SQL is a module for structured data processing.",
"logo": "apache-spark.png",