You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2026/09/18 18:18:49 UTC
(superset) 02/02: fix(datasets): gate partition filter mapping UI by engine capability
This is an automated email from the ASF dual-hosted git repository.
eschutho pushed a commit to branch pfm-7-engine-capability-gate
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 1ce6f4c67849ff412adc31e196d8595285ccc2d4
Author: Elizabeth Thompson <es...@gmail.com>
AuthorDate: Fri Sep 18 18:17:40 2026 +0000
fix(datasets): gate partition filter mapping UI by engine capability
PR #43891 added `partition_value_transform_default` on the engine spec so
the dataset editor can pre-fill a temporal column's value transform, and
set it only on Hive/Impala (Spark inheriting from Hive). That gates *which
transform to suggest*, not *whether the feature applies to the engine at
all*: the editor still offered the "Partition column" dropdown on every
database whenever the `PARTITION_FILTER_MAPPING` flag was on, and
`validate_partition_mapping` only used the engine to check the transform
text parsed for that dialect -- never to reject the feature on engines
where partition-directory pruning does not exist.
Mirroring a filter onto a physical partition column only prunes work on
engines whose tables are laid out as partition directories (Hive-family).
On Postgres the concept does not exist; on Snowflake micro-partitions are
automatic and invisible; BigQuery and Redshift use different mechanisms
entirely. There the dropdown is clutter that maps to nothing.
Add a `supports_partition_filter_mapping` capability flag on
`BaseEngineSpec` (default `False`), set `True` on the same two specs that
carry `partition_value_transform_default` -- `HiveEngineSpec` and
`ImpalaEngineSpec`, with `SparkEngineSpec` inheriting it from Hive. Thread
it to the client the same way the transform default travels: a read-only
`SqlaTable` property sourced from the engine spec, exposed through
`DatasetRestApi.show_columns`, and added to the frontend datasource types.
The editor's `partitionMappingEnabled` gate now also requires the flag, so
the partition mapping UI only appears on an engine that advertises it.
Scope is kept identical to the existing stack: Hive/Impala/Spark only.
Presto/Trino and everything else stay `False`, matching the decision that
left `partition_value_transform_default` off Presto.
Co-Authored-By: Claude Opus 4.8 <no...@anthropic.com>
---
.../DatasourceEditor/DatasourceEditor.tsx | 7 ++-
.../tests/DatasourceEditor.test.tsx | 56 +++++++++++++++++++++-
superset-frontend/src/features/datasets/types.ts | 3 ++
superset/connectors/sqla/models.py | 13 +++++
superset/datasets/api.py | 3 ++
superset/db_engine_specs/base.py | 11 +++++
superset/db_engine_specs/hive.py | 3 ++
superset/db_engine_specs/impala.py | 3 ++
.../partition_mapping_serialization_test.py | 10 ++++
tests/unit_tests/db_engine_specs/test_hive.py | 22 +++++++++
tests/unit_tests/db_engine_specs/test_impala.py | 20 ++++++++
11 files changed, 149 insertions(+), 2 deletions(-)
diff --git a/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx b/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx
index 823171ab0f0..d81b9eed450 100644
--- a/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx
+++ b/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx
@@ -218,6 +218,9 @@ interface DatasourceObject {
partition_mapped_column?: string | null;
// Engine-supplied pre-fill for a temporal column's value transform. Read-only.
partition_value_transform_default?: string | null;
+ // Whether the engine's tables are partition-directory laid out. Gates whether
+ // the partition filter mapping UI is offered at all. Engine-supplied, read-only.
+ supports_partition_filter_mapping?: boolean;
template_params?: string;
spatials?: SpatialConfig[];
all_cols?: string[];
@@ -616,7 +619,9 @@ function ColumnCollectionTable({
expandedColumnName,
}: ColumnCollectionTableProps): JSX.Element {
const partitionMappingEnabled =
- isFeatureEnabled(FeatureFlag.PartitionFilterMapping) && Boolean(datasource);
+ isFeatureEnabled(FeatureFlag.PartitionFilterMapping) &&
+ Boolean(datasource) &&
+ Boolean(datasource?.supports_partition_filter_mapping);
const partitionColumn = partitionMappingEnabled
? datasource?.partition_column
: null;
diff --git a/superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditor.test.tsx b/superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditor.test.tsx
index 74ae681285c..ed6d5973180 100644
--- a/superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditor.test.tsx
+++ b/superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditor.test.tsx
@@ -24,7 +24,11 @@ import {
userEvent,
within,
} from 'spec/helpers/testing-library';
-import { DatasourceType, isFeatureEnabled } from '@superset-ui/core';
+import {
+ DatasourceType,
+ FeatureFlag,
+ isFeatureEnabled,
+} from '@superset-ui/core';
import * as getBootstrapData from 'src/utils/getBootstrapData';
import {
createProps,
@@ -888,3 +892,53 @@ test('DatasourceEditor source pins getSQLLabUrl/openOnSqlLab to the makeUrl + op
/import \{ makeUrl, openInNewTab \} from 'src\/utils\/navigationUtils';/,
);
});
+
+test('partition mapping UI is offered when the engine supports it', async () => {
+ // The feature flag alone is not enough: mirroring a filter onto a partition
+ // column only prunes work on partition-directory engines, so the engine has
+ // to advertise the capability before the editor surfaces it.
+ (isFeatureEnabled as jest.Mock).mockImplementation(
+ flag => flag === FeatureFlag.PartitionFilterMapping,
+ );
+
+ const testProps = createProps();
+ await asyncRender({
+ ...testProps,
+ datasource: {
+ ...testProps.datasource,
+ partition_column: 'ds',
+ supports_partition_filter_mapping: true,
+ },
+ });
+
+ await userEvent.click(screen.getByTestId('collection-tab-Columns'));
+
+ expect(await screen.findByTestId('partition-tag')).toBeInTheDocument();
+});
+
+test('partition mapping UI is withheld on an engine that does not support it', async () => {
+ // Same flag, same partition column, but an engine whose tables are not
+ // partition-directory laid out: the dropdown would map to nothing, so it is
+ // never offered even though the flag is on.
+ (isFeatureEnabled as jest.Mock).mockImplementation(
+ flag => flag === FeatureFlag.PartitionFilterMapping,
+ );
+
+ const testProps = createProps();
+ await asyncRender({
+ ...testProps,
+ datasource: {
+ ...testProps.datasource,
+ partition_column: 'ds',
+ supports_partition_filter_mapping: false,
+ },
+ });
+
+ await userEvent.click(screen.getByTestId('collection-tab-Columns'));
+
+ // Wait for the columns tab to render before asserting the tag is absent.
+ expect(
+ await screen.findByPlaceholderText('Search columns by name'),
+ ).toBeInTheDocument();
+ expect(screen.queryByTestId('partition-tag')).not.toBeInTheDocument();
+});
diff --git a/superset-frontend/src/features/datasets/types.ts b/superset-frontend/src/features/datasets/types.ts
index fca5d4481c8..2d946db8245 100644
--- a/superset-frontend/src/features/datasets/types.ts
+++ b/superset-frontend/src/features/datasets/types.ts
@@ -87,6 +87,9 @@ export type DatasetObject = {
always_filter_main_dttm: boolean;
partition_column?: string | null;
partition_mapped_column?: string | null;
+ // Engine-supplied: whether the engine's tables are partition-directory laid
+ // out, which is what gates whether the editor offers partition filter mapping.
+ supports_partition_filter_mapping?: boolean;
type: DatasourceType;
column_formats: Record<string, string>;
datasource_name: string | null;
diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py
index 4eca5269c18..2211a5110cf 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -1900,6 +1900,19 @@ class SqlaTable(
"""
return self.db_engine_spec.partition_value_transform_default
+ @property
+ def supports_partition_filter_mapping(self) -> bool:
+ """
+ Whether the editor should offer partition filter mapping for this dataset.
+
+ A structural property of the engine, not of the dataset: mirroring a
+ filter onto a partition column only prunes work on engines whose tables
+ are laid out as partition directories (Hive-family). The editor reads it
+ off the dataset payload so the dropdown never appears on an engine where
+ the concept does not exist.
+ """
+ return self.db_engine_spec.supports_partition_filter_mapping
+
@property
def partition_filter_mapping_summary(self) -> dict[str, Any] | None:
"""
diff --git a/superset/datasets/api.py b/superset/datasets/api.py
index ec6bf10b125..406d0bd2243 100644
--- a/superset/datasets/api.py
+++ b/superset/datasets/api.py
@@ -341,6 +341,9 @@ class DatasetRestApi(SoftDeleteApiMixin, BaseSupersetModelRestApi):
"columns.type_generic",
# Engine-supplied pre-fill for the editor's value transform input.
"partition_value_transform_default",
+ # Whether the engine's tables are partition-directory laid out, which is
+ # what decides whether the editor offers partition filter mapping.
+ "supports_partition_filter_mapping",
"database.backend",
"database.allow_multi_catalog",
"columns.advanced_data_type",
diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py
index 278cfebd1fb..946f194fbdc 100644
--- a/superset/db_engine_specs/base.py
+++ b/superset/db_engine_specs/base.py
@@ -722,6 +722,17 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# editor offers no pre-fill and the owner writes the transform themselves.
partition_value_transform_default: str | None = None
+ # Whether the dataset editor should offer partition filter mapping at all.
+ # This is a structural property of the engine, not a pre-fill: mapping a
+ # filter onto a physical partition column only prunes work on engines whose
+ # tables are laid out as partition directories (Hive-family). Elsewhere the
+ # concept either does not exist (Postgres) or is handled invisibly by the
+ # engine (Snowflake micro-partitions, BigQuery/Redshift internals), so the
+ # dropdown would be clutter that maps to nothing. Default `False` so an
+ # engine only advertises the feature by opting in, rather than inheriting it
+ # and having to remember to opt out.
+ supports_partition_filter_mapping: bool = False
+
# Does the query id related to the connection?
# The default value is True, which means that the query id is determined when
# the connection is created.
diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py
index e6f13c567f9..6f757f84094 100644
--- a/superset/db_engine_specs/hive.py
+++ b/superset/db_engine_specs/hive.py
@@ -106,6 +106,9 @@ class HiveEngineSpec(PrestoEngineSpec):
# Epoch-integer partition keys are the common Hive layout, and the
# one-argument form takes the bound being mirrored rather than "now".
partition_value_transform_default = "unix_timestamp(:value)"
+ # Hive tables are laid out as partition directories, so mirroring a filter
+ # onto the partition column lets the engine prune them.
+ supports_partition_filter_mapping = True
# Explicitly opt out (overriding the inherited PrestoEngineSpec value):
# Hive/Spark's GROUPING SETS + GROUPING() marker semantics have not been
# verified against this query pattern, so fall back to one query per
diff --git a/superset/db_engine_specs/impala.py b/superset/db_engine_specs/impala.py
index d47546de814..1b242a60983 100644
--- a/superset/db_engine_specs/impala.py
+++ b/superset/db_engine_specs/impala.py
@@ -51,6 +51,9 @@ class ImpalaEngineSpec(BaseEngineSpec):
# Epoch-integer partition keys are the common Impala layout, and the
# one-argument form takes the bound being mirrored rather than "now".
partition_value_transform_default = "unix_timestamp(:value)"
+ # Impala tables are laid out as partition directories, so mirroring a filter
+ # onto the partition column lets the engine prune them.
+ supports_partition_filter_mapping = True
metadata = {
"description": (
diff --git a/tests/unit_tests/datasets/partition_mapping_serialization_test.py b/tests/unit_tests/datasets/partition_mapping_serialization_test.py
index 4a51ea9d2e5..06eec957ca2 100644
--- a/tests/unit_tests/datasets/partition_mapping_serialization_test.py
+++ b/tests/unit_tests/datasets/partition_mapping_serialization_test.py
@@ -369,6 +369,16 @@ def test_the_engine_transform_default_is_readable_over_the_api() -> None:
assert "partition_value_transform_default" in DatasetRestApi.show_columns
+def test_the_engine_capability_flag_is_readable_over_the_api() -> None:
+ """
+ The editor gates the whole partition mapping UI on this; unexposed, it reads
+ back falsy and the feature never appears, even on an engine that supports it.
+ """
+ from superset.datasets.api import DatasetRestApi
+
+ assert "supports_partition_filter_mapping" in DatasetRestApi.show_columns
+
+
def test_the_mapping_summary_is_gated_on_the_feature_flag(app: Flask) -> None:
"""
With the flag off nothing is mirrored, so an "active" summary would have the
diff --git a/tests/unit_tests/db_engine_specs/test_hive.py b/tests/unit_tests/db_engine_specs/test_hive.py
index 47feb1a8113..b78be71f866 100644
--- a/tests/unit_tests/db_engine_specs/test_hive.py
+++ b/tests/unit_tests/db_engine_specs/test_hive.py
@@ -298,3 +298,25 @@ def test_the_partition_transform_default_is_not_inherited_by_presto() -> None:
from superset.db_engine_specs.presto import PrestoEngineSpec
assert PrestoEngineSpec.partition_value_transform_default is None
+
+
+def test_hive_supports_partition_filter_mapping() -> None:
+ """
+ Hive tables are partition-directory laid out, so the editor offers the
+ mapping. Spark SQL inherits from Hive and shares the layout.
+ """
+ from superset.db_engine_specs.hive import HiveEngineSpec
+ from superset.db_engine_specs.spark import SparkEngineSpec
+
+ assert HiveEngineSpec.supports_partition_filter_mapping is True
+ assert SparkEngineSpec.supports_partition_filter_mapping is True
+
+
+def test_partition_filter_mapping_is_not_inherited_by_presto() -> None:
+ """
+ HiveEngineSpec extends PrestoEngineSpec, so pin that the capability did not
+ travel up: Presto/Trino tables are not partition-directory laid out.
+ """
+ from superset.db_engine_specs.presto import PrestoEngineSpec
+
+ assert PrestoEngineSpec.supports_partition_filter_mapping is False
diff --git a/tests/unit_tests/db_engine_specs/test_impala.py b/tests/unit_tests/db_engine_specs/test_impala.py
index 41fcd0b7440..5b2f7bf7526 100644
--- a/tests/unit_tests/db_engine_specs/test_impala.py
+++ b/tests/unit_tests/db_engine_specs/test_impala.py
@@ -192,3 +192,23 @@ def test_engines_without_a_sensible_default_offer_none() -> None:
assert BaseEngineSpec.partition_value_transform_default is None
assert PostgresEngineSpec.partition_value_transform_default is None
+
+
+def test_impala_supports_partition_filter_mapping() -> None:
+ """Impala tables are partition-directory laid out, so the editor offers it."""
+ from superset.db_engine_specs.impala import ImpalaEngineSpec
+
+ assert ImpalaEngineSpec.supports_partition_filter_mapping is True
+
+
+def test_engines_without_partition_directories_do_not_offer_the_mapping() -> None:
+ """
+ Mirroring a filter onto a partition column only prunes work where tables are
+ laid out as partition directories. Elsewhere the dropdown maps to nothing, so
+ an engine only advertises the feature by opting in.
+ """
+ from superset.db_engine_specs.base import BaseEngineSpec
+ from superset.db_engine_specs.postgres import PostgresEngineSpec
+
+ assert BaseEngineSpec.supports_partition_filter_mapping is False
+ assert PostgresEngineSpec.supports_partition_filter_mapping is False