You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/10/13 16:40:16 UTC
(superset) branch master updated: feat(semantic-layers): add safe containment result cache (#42760)
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 8ace079d8de feat(semantic-layers): add safe containment result cache (#42760)
8ace079d8de is described below
commit 8ace079d8de783a63810193f55709fc13a972b2e
Author: Mike Bridge <mi...@preset.io>
AuthorDate: Tue Oct 13 10:40:08 2026 -0600
feat(semantic-layers): add safe containment result cache (#42760)
Co-authored-by: Mike Bridge <mi...@ext.preset.io>
Co-authored-by: Beto Dealmeida <ro...@dealmeida.net>
Co-authored-by: Claude Fable 5 <no...@anthropic.com>
---
.github/actions/change-detector/action.yml | 3 +
.../workflows/superset-python-integrationtest.yml | 50 ++
UPDATING.md | 6 +
docs/admin_docs/configuration/cache.mdx | 36 +
.../extensions/contribution-types.md | 140 +++
docs/static/feature-flags.json | 7 +
docs/static/resources/openapi.json | 11 +
scripts/change_detector.py | 15 +
.../src/superset_core/semantic_layers/layer.py | 63 +-
.../components/CachedLabel/TooltipContent.test.tsx | 25 +
.../src/components/CachedLabel/TooltipContent.tsx | 21 +-
.../src/components/CachedLabel/index.tsx | 14 +-
.../src/components/CachedLabel/types.ts | 3 +
.../superset-ui-core/src/components/index.ts | 6 +-
.../src/query/types/QueryResponse.ts | 8 +
.../src/explore/components/ChartPills.test.tsx | 134 +++
.../src/explore/components/ChartPills.tsx | 36 +-
superset/charts/data/api.py | 5 +-
superset/charts/schemas.py | 12 +
superset/common/query_actions.py | 1 +
superset/common/query_context_factory.py | 1 +
superset/common/query_context_processor.py | 13 +
superset/common/query_object.py | 15 +
superset/common/utils/query_cache_manager.py | 9 +-
superset/config.py | 12 +
superset/constants.py | 39 +
superset/coordination/cache_backend.py | 72 ++
superset/initialization/__init__.py | 88 ++
superset/models/helpers.py | 3 +
superset/semantic_layers/cache.py | 496 ++++++++++
superset/semantic_layers/cache_coordination.py | 240 +++++
superset/semantic_layers/cache_host.py | 248 +++++
superset/semantic_layers/cache_identity.py | 318 +++++++
superset/semantic_layers/cache_policy.py | 394 ++++++++
superset/semantic_layers/cache_repository.py | 450 ++++++++++
superset/semantic_layers/cache_transform.py | 268 ++++++
superset/semantic_layers/cache_types.py | 132 +++
superset/semantic_layers/mapper.py | 111 ++-
superset/superset_typing.py | 2 +
superset/translations/messages.pot | 9 +
superset/utils/cache_manager.py | 17 +
.../semantic_layers/cache_coordination_tests.py | 200 +++++
tests/unit_tests/charts/test_chart_data_api.py | 46 +-
tests/unit_tests/common/test_query_actions.py | 31 +-
.../unit_tests/common/test_query_cache_manager.py | 18 +
.../common/test_query_context_processor.py | 11 +-
.../common/test_query_context_processor_timing.py | 79 ++
tests/unit_tests/config_test.py | 10 +-
.../unit_tests/coordination/test_cache_backend.py | 189 ++++
tests/unit_tests/initialization_test.py | 73 ++
tests/unit_tests/queries/query_object_test.py | 29 +
tests/unit_tests/scripts/change_detector_test.py | 16 +
.../semantic_layers/cache_coordination_test.py | 525 +++++++++++
.../unit_tests/semantic_layers/cache_host_test.py | 369 ++++++++
.../semantic_layers/cache_identity_test.py | 280 ++++++
.../semantic_layers/cache_integration_test.py | 996 +++++++++++++++++++++
.../semantic_layers/cache_policy_test.py | 963 ++++++++++++++++++++
.../semantic_layers/cache_repository_test.py | 921 +++++++++++++++++++
tests/unit_tests/semantic_layers/cache_test.py | 956 ++++++++++++++++++++
.../semantic_layers/cache_transform_test.py | 223 +++++
tests/unit_tests/semantic_layers/conftest.py | 92 ++
tests/unit_tests/semantic_layers/layer_test.py | 91 ++
tests/unit_tests/semantic_layers/mapper_test.py | 87 ++
tests/unit_tests/utils/test_cache_manager.py | 33 +
64 files changed, 9738 insertions(+), 33 deletions(-)
diff --git a/.github/actions/change-detector/action.yml b/.github/actions/change-detector/action.yml
index decc74fc339..192a70ba3b0 100644
--- a/.github/actions/change-detector/action.yml
+++ b/.github/actions/change-detector/action.yml
@@ -20,6 +20,9 @@ outputs:
superset-extensions-cli:
description: Whether superset-extensions-cli package-related files were changed
value: ${{ steps.change-detector.outputs.superset-extensions-cli }}
+ semantic-layers:
+ description: Whether semantic-layer files were changed
+ value: ${{ steps.change-detector.outputs.semantic-layers }}
languages:
description: >-
JSON array of CodeQL languages touched, classified by file extension
diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
index 4917f0dc2b8..0fee68c9fc3 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -27,6 +27,7 @@ jobs:
pull-requests: read
outputs:
python: ${{ steps.check.outputs.python }}
+ semantic-layers: ${{ steps.check.outputs.semantic-layers }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -38,6 +39,55 @@ jobs:
with:
token: ${{ secrets.GITHUB_TOKEN }}
+ test-semantic-cache-coordination:
+ needs: changes
+ if: needs.changes.outputs.semantic-layers == 'true'
+ runs-on: ubuntu-26.04
+ timeout-minutes: 15
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SEMANTIC_CACHE_REDIS_PORT: 16379
+ SEMANTIC_CACHE_SENTINEL_PORT: 26379
+ services:
+ redis:
+ image: redis:7.4.10-alpine3.21
+ ports:
+ - 16379:6379
+ options: >-
+ --health-cmd="redis-cli ping"
+ --health-interval=2s
+ --health-timeout=2s
+ --health-retries=20
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ submodules: recursive
+ - name: Setup Python
+ uses: $/.github/actions/setup-backend/
+ - name: Start pinned Redis Sentinel
+ run: |
+ docker run --detach --name semantic-cache-sentinel \
+ --publish 26379:26379 \
+ --add-host host.docker.internal:host-gateway \
+ redis:7.4.10-alpine3.21 sh -c \
+ 'printf "port 26379\nsentinel resolve-hostnames yes\nsentinel monitor mymaster host.docker.internal 16379 1\nsentinel down-after-milliseconds mymaster 5000\nsentinel failover-timeout mymaster 10000\n" > /tmp/sentinel.conf && exec redis-server /tmp/sentinel.conf --sentinel'
+ for attempt in $(seq 1 30); do
+ if docker exec semantic-cache-sentinel redis-cli -p 26379 ping; then
+ exit 0
+ fi
+ sleep 1
+ done
+ exit 1
+ - name: Run semantic-cache Redis coordination tests
+ timeout-minutes: 5
+ run: |
+ pytest -q tests/integration_tests/semantic_layers/cache_coordination_tests.py
+ - name: Stop Redis Sentinel
+ if: always()
+ run: docker rm --force semantic-cache-sentinel || true
+
test-mysql:
needs: changes
if: needs.changes.outputs.python == 'true'
diff --git a/UPDATING.md b/UPDATING.md
index 0d5e754bdeb..177ffcad031 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -24,6 +24,11 @@ assists people when migrating to a new version.
## Next
+- `SEMANTIC_LAYER_CONTAINMENT_CACHE` is off by default; opted-in providers can
+ reuse cached semantic-view results for narrower queries. Containment keys include
+ the view's `metadata_cache_token`, so metadata refresh makes old entries miss. Enable
+ `SEMANTIC_LAYERS`; configure shared persistent `DATA_CACHE_CONFIG` and Redis-backed `DISTRIBUTED_COORDINATION_CONFIG` on every web/worker process.
+
- Address validation refuses additional non-global address forms. Addresses in
the well-known NAT64 prefix `64:ff9b::/96` are checked using their embedded IPv4
address and refused when that address is unsafe. Local-use translation
@@ -59,6 +64,7 @@ assists people when migrating to a new version.
MCP chart tools that accept `extra_form_data` (chart data, SQL and previews, for
saved and unsaved charts). This changes chart cache keys for dashboard-filtered
queries once, so expect a cold chart cache right after deploying.
+
- Semantic-layer configuration Save and Reload metadata increment a database-backed
cache generation. Apply the `c179af013f48` migration before deploying the updated
application; existing semantic chart caches become cold once on upgrade.
diff --git a/docs/admin_docs/configuration/cache.mdx b/docs/admin_docs/configuration/cache.mdx
index 855bb13c10b..4ba72ee0609 100644
--- a/docs/admin_docs/configuration/cache.mdx
+++ b/docs/admin_docs/configuration/cache.mdx
@@ -7,6 +7,42 @@ version: 1
# Caching
+## Semantic containment cache
+
+`SEMANTIC_LAYER_CONTAINMENT_CACHE` is experimental and default-off. Enabling
+the flag alone is insufficient: enable `SEMANTIC_LAYERS`, use a provider that
+explicitly delegates containment caching, and configure persistent shared
+`DATA_CACHE_CONFIG` plus Redis-backed `DISTRIBUTED_COORDINATION_CONFIG` on every
+web and worker process. The default `NullCache` data backend disables containment.
+
+See the [semantic containment setup and validation guide](/developer-docs/extensions/contribution-types#semantic-result-containment-caching)
+for the two-backend example, supported backends, timeout semantics, emitted
+metrics and concurrency limitations. Restart every web/worker process after
+configuration changes and verify the effective-state metric before a canary.
+The reader fence uses identity format `v4` and matches a per-write nonce in each
+descriptor/result pair. Existing `v3` entries become cold misses; they need no
+manual deletion. During a mixed-version rollout, old workers still use the old
+unfenced path, so finish the web/worker rollout before validating the fence.
+Monitor `semantic_cache.containment.fence_reject` alongside misses and coordination
+failures during forced-refresh testing: rejected pairs must fall back to another
+valid candidate or the provider, never serve the stale writer's mismatched value.
+Pruning removes invalid descriptors after rechecking the current pair and leaves
+payloads to their configured TTL. Preserve the validation hold until the
+split-store process-pause scenario and provider-result comparisons pass.
+
+Filtering and rollup preserve nullable integer, decimal and timestamp values;
+unsupported conversions or overflowing aggregates fall back to the provider.
+The ORM view's nonempty host-supplied `metadata_cache_token` participates in
+containment identity. Missing, empty, or invalid host tokens bypass containment
+and execute the provider, even if older entries exist. A provider's optional token
+cannot substitute for the host generation. Containment requires the host metadata
+cache-generation binding; no metadata-refresh adapter is required.
+
+To roll back, disable `SEMANTIC_LAYER_CONTAINMENT_CACHE`, restart every process,
+and confirm `semantic_cache.containment.enabled` is zero and provider queries
+continue. This best-effort cache's fallback behavior does not change the other
+cache backends' availability requirements below.
+
:::note
When a cache backend is configured, Superset expects it to remain available. Operations will
fail if the configured backend becomes unavailable rather than silently degrading. This
diff --git a/docs/developer_docs/extensions/contribution-types.md b/docs/developer_docs/extensions/contribution-types.md
index 416df699d27..e317c248cf6 100644
--- a/docs/developer_docs/extensions/contribution-types.md
+++ b/docs/developer_docs/extensions/contribution-types.md
@@ -338,6 +338,146 @@ Custom SQL modes are visible but disabled, so users cannot build an expression
the backend would reject. Unknown feature strings and payloads with no
`semantic_view_features` field are ignored, preserving existing behavior.
+#### Semantic result containment caching
+
+Superset containment caching is experimental and off by default. A provider must
+explicitly opt in; the safe defaults leave caching with the provider and scope
+results to an execution context:
+
+```python
+from superset_core.semantic_layers.layer import (
+ SemanticCacheCapabilities,
+ SemanticCacheExecutionContext,
+ SemanticCacheIdentityMaterial,
+ SemanticCacheResponsibility,
+ SemanticCacheScope,
+)
+
+class MySemanticLayer(SemanticLayer[MyConfig, MySemanticView]):
+ semantic_cache_responsibility = SemanticCacheResponsibility.SUPERSET
+ semantic_cache_scope = SemanticCacheScope.EXECUTION_CONTEXT
+ semantic_cache_capabilities = SemanticCacheCapabilities(
+ comparisons=True,
+ membership=True,
+ nulls=True,
+ pattern_escape="\\",
+ )
+
+ def get_semantic_cache_provider_identity(self) -> SemanticCacheIdentityMaterial:
+ return SemanticCacheIdentityMaterial(
+ {"provider_version": "v1", "catalog": self.config.catalog}
+ )
+
+ def get_semantic_cache_context_identity(
+ self,
+ context: SemanticCacheExecutionContext,
+ ) -> SemanticCacheIdentityMaterial:
+ return SemanticCacheIdentityMaterial({"tenant": self.tenant_id(context)})
+```
+
+Identity material must be secret-free and include every provider setting that can
+change results. For execution-context scope, Superset also hashes the principal,
+roles, guest-token claims, and row-level-security cache key. Returning `None` from
+either identity method bypasses containment. Use `GLOBAL` only when results are
+provably identical across principals and tenants; containment is bypassed for a
+`GLOBAL` view whenever Superset row-level security applies to the request, since
+that variation is invisible to the provider. Declare only filter capabilities
+whose provider semantics exactly match Superset's post-processing semantics.
+
+`comparisons=True` assumes binary (codepoint) string ordering and equality;
+`pattern_escape` assumes case-sensitive SQL LIKE. Providers using case-insensitive
+collations must not opt into these capabilities. These declarations apply to
+every string dimension exposed by the provider, not just the queried example.
+
+Narrowing a time range on a grained axis always requires a provider round-trip:
+leftover WHERE filters cannot be applied to cached bucket starts, including when
+the filter names the raw column of that axis.
+
+Operators enable both `SEMANTIC_LAYERS` and the development feature flag
+`SEMANTIC_LAYER_CONTAINMENT_CACHE`, and configure two backends:
+
+- `DATA_CACHE_CONFIG` holds the cached results and their descriptors. It must be
+ a persistent cache shared by every web and worker process, such as `RedisCache`.
+ The default `NullCache` discards every value, so containment would only ever
+ miss; `RedisSentinelCache` reads from replicas that can lag the master. Both
+ disable containment at startup rather than run it ineffectively.
+- `DISTRIBUTED_COORDINATION_CONFIG` must select `RedisCache` or
+ `RedisSentinelCache`; containment requires its atomic owner-token lease
+ operations.
+
+```python
+DATA_CACHE_CONFIG = {
+ "CACHE_TYPE": "RedisCache",
+ "CACHE_DEFAULT_TIMEOUT": 86400,
+ "CACHE_KEY_PREFIX": "superset_results",
+ "CACHE_REDIS_URL": "redis://redis:6379/1",
+}
+DISTRIBUTED_COORDINATION_CONFIG = {
+ "CACHE_TYPE": "RedisCache",
+ "CACHE_REDIS_HOST": "redis",
+ "CACHE_REDIS_PORT": 6379,
+ "CACHE_REDIS_DB": 2,
+}
+```
+
+Containment follows the same cache timeout the ordinary result cache resolves for
+a request (custom, then chart, then dataset), so a chart-level timeout of `-1`
+bypasses containment entirely, and honors `DATA_CACHE_MAX_VALUE_SIZE`: a result
+larger than that many bytes is served but not stored. The bounded defaults are
+`SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS = 1.0` and
+`SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS = 30`. A missing or unsuitable backend
+and missing or invalid coordination each disable only containment, log a fixed
+warning without configuration or query content, and leave provider execution
+available.
+
+Roll out to a canary worker cohort after recording provider latency, error rate,
+and cache-backend health for a comparable baseline. Observe at least one normal
+traffic cycle. Alert ownership should sit with the semantic-platform operator.
+Track `semantic_cache.containment.enabled`, `semantic_cache.containment.hit`,
+`semantic_cache.containment.miss`, `semantic_cache.containment.bypass`,
+`semantic_cache.containment.store_skipped`, `semantic_cache.containment.store_failure`,
+`semantic_cache.containment.lookup_failure`, `semantic_cache.containment.transform_failure`,
+`semantic_cache.containment.prune_failure`, `semantic_cache.containment.fence_reject`,
+`semantic_cache.containment.coordination_failure`,
+`semantic_cache.containment.unsupported`, and `semantic_cache.containment.invalid_configuration`.
+Alert on the `semantic_cache.containment.store_failure`,
+`semantic_cache.containment.lookup_failure`, `semantic_cache.containment.transform_failure`,
+`semantic_cache.containment.prune_failure`, and `semantic_cache.containment.coordination_failure`
+counters, and unexpected `semantic_cache.containment.unsupported` or
+`semantic_cache.containment.invalid_configuration` at startup.
+Coordination socket/connect timeouts derive from `SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS`
+(with a 1 ms minimum for zero wait); these bound individual I/O calls, not the whole request.
+Separate lease/value clients use a non-atomic immediate ownership recheck before writing.
+The atomic Lua fence is a same-client optimisation for explicitly injected clients.
+Default application initialization constructs a private coordination client, so
+ordinary deployments do not exercise that optimisation even when both caches
+point at the same Redis server. Reader-side fencing protects this split-store
+path: each store writes a fresh nonce into both the descriptor and a result
+envelope. Exact-key and containment lookups accept only matching nonempty nonces.
+A stale SET after lease takeover can overwrite a payload, but a mismatched pair
+is rejected rather than served. The lookup may use another valid candidate or
+execute the provider. `semantic_cache.containment.fence_reject` counts lookups
+that rejected at least one invalid publication (not ordinary expired values).
+
+Maintenance rechecks the current descriptor and payload under the mutation lease
+before pruning. A concurrently repaired matching pair is retained. Rejection
+pruning never deletes payloads, which expire according to their own TTL. This
+fence can trade cache hits for misses under a stalled writer; it is not a
+cross-store transaction or a promise of linearizable refreshes. Store deduplication
+also requires a matching pair, so an invalid payload cannot suppress its repair.
+The cache identity format is `v4`; pre-fence `v3` entries are not reused.
+Validate the split-store path under concurrent forced refreshes, whole-process
+pauses exceeding the lease, and narrowed time ranges.
+Define deployment-specific rollback thresholds before the
+canary; recommended triggers are any provider-result mismatch, sustained provider
+error regression, coordination failures above 1% of cache mutations, or lookup and
+store failures above 5% of semantic requests.
+
+Rollback by disabling `SEMANTIC_LAYER_CONTAINMENT_CACHE` and restarting every web
+and worker process. Confirm worker convergence with an `enabled` gauge of zero for
+each process and normal provider-query success. Cache data need not be deleted for
+rollback because disabled processes do not read or write it.
+
#### Choosing a default time dimension
A semantic view may set `preferred_temporal_dimension` to the name of a temporal
diff --git a/docs/static/feature-flags.json b/docs/static/feature-flags.json
index 75a60e15e36..580753c01ae 100644
--- a/docs/static/feature-flags.json
+++ b/docs/static/feature-flags.json
@@ -99,6 +99,13 @@
"lifecycle": "development",
"description": "Enable semantic layers and show semantic views alongside datasets"
},
+ {
+ "name": "SEMANTIC_LAYER_CONTAINMENT_CACHE",
+ "default": false,
+ "lifecycle": "development",
+ "description": "Enable Superset-owned containment caching for opted-in semantic providers. Requires shared DATA_CACHE_CONFIG and Redis-backed DISTRIBUTED_COORDINATION_CONFIG.",
+ "docs": "https://superset.apache.org/admin-docs/configuration/cache/#semantic-containment-cache"
+ },
{
"name": "TABLE_V2_TIME_COMPARISON_ENABLED",
"default": false,
diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json
index d29df95aa1b..5b18ba4cc7b 100644
--- a/docs/static/resources/openapi.json
+++ b/docs/static/resources/openapi.json
@@ -1991,6 +1991,17 @@
"description": "Amount of rows in result set",
"type": "integer"
},
+ "semantic_cache_status": {
+ "description": "Semantic containment cache provenance of this query: HIT when every semantic query behind it was served from containment, MISS when none was, MIXED when only some were (for example a time-comparison offset).",
+ "enum": [
+ "HIT",
+ "MISS",
+ "MIXED",
+ null
+ ],
+ "nullable": true,
+ "type": "string"
+ },
"stacktrace": {
"description": "Stacktrace if there was an error",
"nullable": true,
diff --git a/scripts/change_detector.py b/scripts/change_detector.py
index 18e631e9120..b3dc2fd7d11 100755
--- a/scripts/change_detector.py
+++ b/scripts/change_detector.py
@@ -64,6 +64,21 @@ PATTERNS = {
r"^\.github/actions/",
r"^superset-frontend/",
],
+ # Scopes the semantic-cache coordination job (real Redis/Sentinel services)
+ # to changes that can affect it, instead of every python-touching PR.
+ "semantic-layers": [
+ r"^\.github/workflows/superset-python-integrationtest\.yml$",
+ r"^\.github/actions/change-detector/",
+ r"^scripts/change_detector\.py$",
+ r"^superset/semantic_layers/",
+ # The owner-token lease operations the job exercises against real
+ # Redis/Sentinel live outside the semantic_layers package.
+ r"^superset/coordination/cache_backend\.py$",
+ r"^superset/utils/cache_manager\.py$",
+ r"^superset-core/src/superset_core/semantic_layers/",
+ r"^tests/unit_tests/semantic_layers/",
+ r"^tests/integration_tests/semantic_layers/",
+ ],
"docker": [
r"^Dockerfile$",
r"^docker.*",
diff --git a/superset-core/src/superset_core/semantic_layers/layer.py b/superset-core/src/superset_core/semantic_layers/layer.py
index bbc85ced8e3..93bbb6297b7 100644
--- a/superset-core/src/superset_core/semantic_layers/layer.py
+++ b/superset-core/src/superset_core/semantic_layers/layer.py
@@ -17,8 +17,10 @@
from __future__ import annotations
+import enum
from abc import ABC, abstractmethod
-from typing import Any, ClassVar, Generic, TypeVar
+from dataclasses import dataclass
+from typing import Any, ClassVar, Generic, Mapping, TypeVar
from pydantic import BaseModel
@@ -28,12 +30,71 @@ ConfigT = TypeVar("ConfigT", bound=BaseModel)
SemanticViewT = TypeVar("SemanticViewT", bound="SemanticView")
+class SemanticCacheResponsibility(str, enum.Enum):
+ """Component responsible for caching semantic provider results."""
+
+ PROVIDER = "provider"
+ SUPERSET = "superset"
+
+
+class SemanticCacheScope(str, enum.Enum):
+ """Execution scope in which a semantic result may be reused."""
+
+ EXECUTION_CONTEXT = "execution_context"
+ GLOBAL = "global"
+
+
+@dataclass(frozen=True)
+class SemanticCacheCapabilities:
+ """Provider guarantees for equivalent cached post-processing semantics."""
+
+ comparisons: bool = False
+ membership: bool = False
+ nulls: bool = False
+ pattern_escape: str | None = None
+
+
+@dataclass(frozen=True)
+class SemanticCacheExecutionContext:
+ """Typed host principal context offered to provider identity policy."""
+
+ principal_id: str | None
+ role_ids: tuple[str, ...]
+ host_identity: str
+
+
+@dataclass(frozen=True)
+class SemanticCacheIdentityMaterial:
+ """Explicit secret-free, result-affecting provider identity material."""
+
+ values: Mapping[str, object]
+
+
class SemanticLayer(ABC, Generic[ConfigT, SemanticViewT]):
"""
Abstract base class for semantic layers.
"""
configuration_class: type[BaseModel]
+ semantic_cache_responsibility: SemanticCacheResponsibility = (
+ SemanticCacheResponsibility.PROVIDER
+ )
+ semantic_cache_scope: SemanticCacheScope = SemanticCacheScope.EXECUTION_CONTEXT
+ semantic_cache_capabilities: SemanticCacheCapabilities = SemanticCacheCapabilities()
+
+ def get_semantic_cache_provider_identity(
+ self,
+ ) -> SemanticCacheIdentityMaterial | None:
+ """Return explicit secret-free provider identity, or bypass containment."""
+ return None
+
+ def get_semantic_cache_context_identity(
+ self,
+ context: SemanticCacheExecutionContext,
+ ) -> SemanticCacheIdentityMaterial | None:
+ """Return complete request-context identity, or bypass containment."""
+ return None
+
# Opt in only when all result methods enforce the advertised guarantee.
# None preserves legacy result-cache keys without constructing the provider.
result_cache_version: ClassVar[str | None] = None
diff --git a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.test.tsx b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.test.tsx
index 72629c868b2..ebd87cb4c47 100644
--- a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.test.tsx
+++ b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.test.tsx
@@ -36,3 +36,28 @@ test('Rendering TooltipContent correctly - with timestep', () => {
.fromNow()}. Click to force-refresh`,
);
});
+
+test('Rendering TooltipContent for semantic cache', () => {
+ render(<TooltipContent cacheSource="semantic" />);
+ expect(screen.getByTestId('tooltip-content')).toHaveTextContent(
+ 'Loaded from semantic cache. Click to force-refresh',
+ );
+});
+
+test('Rendering timestamped TooltipContent for semantic cache', () => {
+ render(
+ <TooltipContent cachedTimestamp="01-01-2000" cacheSource="semantic" />,
+ );
+ expect(screen.getByTestId('tooltip-content')?.textContent).toBe(
+ `Loaded from semantic cache ${extendedDayjs
+ .utc('01-01-2000')
+ .fromNow()}. Click to force-refresh`,
+ );
+});
+
+test('Rendering TooltipContent for ordinary cache', () => {
+ render(<TooltipContent cacheSource="result" />);
+ expect(screen.getByTestId('tooltip-content')).toHaveTextContent(
+ 'Loaded from cache. Click to force-refresh',
+ );
+});
diff --git a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.tsx b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.tsx
index da85ff5cd06..5689a198397 100644
--- a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.tsx
+++ b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/TooltipContent.tsx
@@ -20,18 +20,33 @@
import { FC } from 'react';
import { t } from '@apache-superset/core/translation';
import { extendedDayjs } from '../../utils/dates';
+import type { CacheSource } from './types';
interface Props {
cachedTimestamp?: string;
+ cacheSource?: CacheSource;
}
-export const TooltipContent: FC<Props> = ({ cachedTimestamp }) => {
+export const TooltipContent: FC<Props> = ({
+ cachedTimestamp,
+ cacheSource = 'result',
+}) => {
+ const loadedText =
+ cacheSource === 'semantic'
+ ? {
+ timestamped: t('Loaded from semantic cache'),
+ untimed: t('Loaded from semantic cache'),
+ }
+ : {
+ timestamped: t('Loaded data cached'),
+ untimed: t('Loaded from cache'),
+ };
const cachedText = cachedTimestamp ? (
<span>
- {t('Loaded data cached')}
+ {loadedText.timestamped}
<b> {extendedDayjs.utc(cachedTimestamp).fromNow()}</b>
</span>
) : (
- t('Loaded from cache')
+ loadedText.untimed
);
return (
diff --git a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/index.tsx b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/index.tsx
index 027526fe33f..6acfc97905c 100644
--- a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/index.tsx
+++ b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/index.tsx
@@ -23,19 +23,25 @@ import { Icons } from '@superset-ui/core/components/Icons';
import { Label } from '../Label';
import { Tooltip } from '../Tooltip';
import { TooltipContent } from './TooltipContent';
-import type { CacheLabelProps } from './types';
+import type { CacheLabelProps, CacheSource } from './types';
export const CachedLabel: FC<CacheLabelProps> = ({
className,
onClick,
cachedTimestamp,
+ cacheSource = 'result',
}) => {
const [hovered, setHovered] = useState(false);
const labelType = hovered ? 'info' : 'default';
return (
<Tooltip
- title={<TooltipContent cachedTimestamp={cachedTimestamp} />}
+ title={
+ <TooltipContent
+ cachedTimestamp={cachedTimestamp}
+ cacheSource={cacheSource}
+ />
+ }
id="cache-desc-tooltip"
>
<Label
@@ -46,10 +52,10 @@ export const CachedLabel: FC<CacheLabelProps> = ({
onMouseOut={() => setHovered(false)}
icon={<Icons.SyncOutlined iconSize="m" />}
>
- {t('Cached')}
+ {cacheSource === 'semantic' ? t('Semantic cache') : t('Cached')}
</Label>
</Tooltip>
);
};
-export type { CacheLabelProps };
+export type { CacheLabelProps, CacheSource };
diff --git a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/types.ts b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/types.ts
index 5a8b3567405..91c2390ddcb 100644
--- a/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/types.ts
+++ b/superset-frontend/packages/superset-ui-core/src/components/CachedLabel/types.ts
@@ -19,8 +19,11 @@
import type { MouseEventHandler } from 'react';
+export type CacheSource = 'result' | 'semantic';
+
export interface CacheLabelProps {
onClick?: MouseEventHandler<HTMLElement>;
cachedTimestamp?: string;
+ cacheSource?: CacheSource;
className?: string;
}
diff --git a/superset-frontend/packages/superset-ui-core/src/components/index.ts b/superset-frontend/packages/superset-ui-core/src/components/index.ts
index a09ba2ab0ae..3a6f39ea0e2 100644
--- a/superset-frontend/packages/superset-ui-core/src/components/index.ts
+++ b/superset-frontend/packages/superset-ui-core/src/components/index.ts
@@ -65,7 +65,11 @@ export { Badge, type BadgeProps } from './Badge';
export { Button, type ButtonProps, type OnClickHandler } from './Button';
export { ButtonGroup, type ButtonGroupProps } from './ButtonGroup';
export { Breadcrumb, type BreadcrumbProps } from './Breadcrumb';
-export { CachedLabel, type CacheLabelProps } from './CachedLabel';
+export {
+ CachedLabel,
+ type CacheLabelProps,
+ type CacheSource,
+} from './CachedLabel';
export { Card } from './Card';
export {
Checkbox,
diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/QueryResponse.ts b/superset-frontend/packages/superset-ui-core/src/query/types/QueryResponse.ts
index 6152beb3ae5..42fe19df541 100644
--- a/superset-frontend/packages/superset-ui-core/src/query/types/QueryResponse.ts
+++ b/superset-frontend/packages/superset-ui-core/src/query/types/QueryResponse.ts
@@ -25,6 +25,13 @@ import { AnnotationData } from './AnnotationLayer';
*/
export type DataRecordValue = number | string | boolean | Date | null | bigint;
+/**
+ * Semantic containment-cache provenance of one query: HIT when every semantic
+ * query behind it (main plus time offsets) was served from containment, MISS
+ * when none was, MIXED when only some were.
+ */
+export type SemanticCacheStatus = 'HIT' | 'MISS' | 'MIXED';
+
export interface DataRecord {
[key: string]: DataRecordValue;
}
@@ -73,6 +80,7 @@ export interface ChartDataResponseResult {
coltypes: GenericDataType[];
error: string | null;
is_cached: boolean;
+ semantic_cache_status?: SemanticCacheStatus | null;
query: string;
rowcount: number;
sql_rowcount: number;
diff --git a/superset-frontend/src/explore/components/ChartPills.test.tsx b/superset-frontend/src/explore/components/ChartPills.test.tsx
new file mode 100644
index 00000000000..7a30aaeba23
--- /dev/null
+++ b/superset-frontend/src/explore/components/ChartPills.test.tsx
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { fireEvent, render, screen } from 'spec/helpers/testing-library';
+import type { QueryData } from '@superset-ui/core';
+import { ChartPills } from './ChartPills';
+
+jest.mock('@superset-ui/core/components', () => ({
+ Button: jest.requireActual('@superset-ui/core/components/Button').Button,
+ CachedLabel: ({ cacheSource }: { cacheSource?: string }) => (
+ <span data-test="cached-label">
+ {cacheSource === 'semantic' ? 'Semantic cache' : 'Ordinary cache'}
+ </span>
+ ),
+ Timer: () => <span data-test="timer" />,
+}));
+
+const response = (
+ semanticCacheStatus: QueryData['semantic_cache_status'],
+ isCached = true,
+): QueryData =>
+ ({
+ is_cached: isCached,
+ semantic_cache_status: semanticCacheStatus,
+ }) as QueryData;
+
+test('passes semantic cache source to the cached label', () => {
+ render(
+ <ChartPills
+ queriesResponse={[response('HIT')]}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={jest.fn()}
+ hideRowCount
+ />,
+ );
+
+ expect(screen.getByTestId('cached-label')).toHaveTextContent(
+ 'Semantic cache',
+ );
+});
+
+test('keeps ordinary cache source distinct', () => {
+ render(
+ <ChartPills
+ queriesResponse={[response('MISS')]}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={jest.fn()}
+ hideRowCount
+ />,
+ );
+
+ expect(screen.getByTestId('cached-label')).toHaveTextContent(
+ 'Ordinary cache',
+ );
+});
+
+test('shows the cache label for a direct semantic cache hit', () => {
+ render(
+ <ChartPills
+ queriesResponse={[response('HIT', false)]}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={jest.fn()}
+ hideRowCount
+ />,
+ );
+
+ expect(screen.getByTestId('cached-label')).toHaveTextContent(
+ 'Semantic cache',
+ );
+});
+
+test('does not claim a cache hit for mixed semantic provenance', () => {
+ render(
+ <ChartPills
+ queriesResponse={[response('MIXED', false)]}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={jest.fn()}
+ hideRowCount
+ />,
+ );
+
+ expect(screen.queryByTestId('cached-label')).not.toBeInTheDocument();
+});
+
+test.each(['MISS', undefined] as const)(
+ 'aggregates a data HIT and a row-count %s as mixed provenance',
+ countStatus => {
+ const refresh = jest.fn();
+ render(
+ <ChartPills
+ queriesResponse={[
+ response('HIT', false),
+ { ...response(countStatus, false), data: [{ rowcount: 100 }] },
+ ]}
+ formData={{ viz_type: 'table', server_pagination: true }}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={refresh}
+ hideRowCount
+ />,
+ );
+ expect(screen.getByText('Mixed cache')).toBeInTheDocument();
+ expect(screen.queryByText('Semantic cache')).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button', { name: 'Mixed cache' }));
+ expect(refresh).toHaveBeenCalledTimes(1);
+ },
+);
+
+test('aggregates all semantic hits', () => {
+ render(
+ <ChartPills
+ queriesResponse={[response('HIT', false), response('HIT', false)]}
+ chartUpdateStartTime={0}
+ refreshCachedQuery={jest.fn()}
+ hideRowCount
+ />,
+ );
+ expect(screen.getByText('Semantic cache')).toBeInTheDocument();
+});
diff --git a/superset-frontend/src/explore/components/ChartPills.tsx b/superset-frontend/src/explore/components/ChartPills.tsx
index da7de6fc8b4..75dd10324bb 100644
--- a/superset-frontend/src/explore/components/ChartPills.tsx
+++ b/superset-frontend/src/explore/components/ChartPills.tsx
@@ -18,8 +18,10 @@
*/
import { forwardRef, RefObject } from 'react';
import { QueryData, VizType } from '@superset-ui/core';
+import { t } from '@apache-superset/core/translation';
import { css, SupersetTheme } from '@apache-superset/core/theme';
import {
+ Button,
CachedLabel,
type LabelType,
Timer,
@@ -66,6 +68,17 @@ export const ChartPills = forwardRef(
) => {
const isLoading = chartStatus === 'loading';
const firstQueryResponse = queriesResponse?.[0];
+ const hasSemanticHit = queriesResponse?.some(response =>
+ ['HIT', 'MIXED'].includes(response.semantic_cache_status ?? ''),
+ );
+ const hasSemanticMiss = queriesResponse?.some(
+ response => response.semantic_cache_status !== 'HIT',
+ );
+ const semanticCacheStatus = hasSemanticHit
+ ? hasSemanticMiss
+ ? 'MIXED'
+ : 'HIT'
+ : 'MISS';
// For table charts with server pagination, check second query for total count
const isTableChart =
@@ -100,11 +113,26 @@ export const ChartPills = forwardRef(
limit={Number(rowLimit ?? 0)}
/>
)}
- {!isLoading && firstQueryResponse?.is_cached && (
- <CachedLabel
+ {!isLoading &&
+ semanticCacheStatus !== 'MIXED' &&
+ (firstQueryResponse?.is_cached ||
+ semanticCacheStatus === 'HIT') && (
+ <CachedLabel
+ onClick={refreshCachedQuery}
+ cachedTimestamp={firstQueryResponse?.cached_dttm}
+ cacheSource={
+ semanticCacheStatus === 'HIT' ? 'semantic' : 'result'
+ }
+ />
+ )}
+ {!isLoading && semanticCacheStatus === 'MIXED' && (
+ <Button
+ buttonStyle="link"
+ buttonSize="small"
onClick={refreshCachedQuery}
- cachedTimestamp={firstQueryResponse.cached_dttm}
- />
+ >
+ {t('Mixed cache')}
+ </Button>
)}
<Timer
startTime={chartUpdateStartTime}
diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py
index 9e386f1638d..020e843e85e 100644
--- a/superset/charts/data/api.py
+++ b/superset/charts/data/api.py
@@ -51,7 +51,7 @@ from superset.common.chart_data import ChartDataResultFormat, ChartDataResultTyp
from superset.common.chart_data_timing import ChartDataExecutionResult
from superset.common.query_context_factory import QueryContextFactory
from superset.connectors.sqla.models import BaseDatasource
-from superset.constants import CACHE_DISABLED_TIMEOUT
+from superset.constants import CACHE_DISABLED_TIMEOUT, SemanticCacheStatus
from superset.daos.exceptions import DatasourceNotFound
from superset.exceptions import QueryObjectValidationError, SupersetSecurityException
from superset.extensions import cache_manager, event_logger
@@ -589,6 +589,9 @@ class ChartDataRestApi(ChartRestApi):
)
resp = make_response(response_data, 200)
resp.headers["Content-Type"] = "application/json; charset=utf-8"
+ resp.headers["X-Superset-Semantic-Cache"] = SemanticCacheStatus.combine(
+ query.get("semantic_cache_status") for query in queries
+ ).value
return resp
return self.response_400(message=f"Unsupported result_format: {result_format}")
diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py
index d169f611207..11478d228d9 100644
--- a/superset/charts/schemas.py
+++ b/superset/charts/schemas.py
@@ -37,6 +37,7 @@ from marshmallow_union import Union
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
from superset.common.chart_data_timing import CHART_DATA_TIMING_VERSION
+from superset.constants import SemanticCacheStatus
from superset.db_engine_specs.base import builtin_time_grains
from superset.semantic_layers.bundle_schemas import SemanticViewReferenceSchema
from superset.subjects.schemas import SubjectResponseSchema
@@ -1838,6 +1839,17 @@ class ChartDataResponseResult(Schema):
required=True,
allow_none=None,
)
+ semantic_cache_status: fields.String = fields.String(
+ metadata={
+ "description": "Semantic containment cache provenance of this "
+ "query: HIT when every semantic query behind it was served from "
+ "containment, MISS when none was, MIXED when only some were "
+ "(for example a time-comparison offset)."
+ },
+ validate=validate.OneOf([status.value for status in SemanticCacheStatus]),
+ required=False,
+ allow_none=True,
+ )
query = fields.String(
metadata={
"description": "The executed query statement. May be absent when "
diff --git a/superset/common/query_actions.py b/superset/common/query_actions.py
index 1591e92338c..71955cc80b1 100644
--- a/superset/common/query_actions.py
+++ b/superset/common/query_actions.py
@@ -260,6 +260,7 @@ def _materialize_full_payload(
"rowcount": payload.get("rowcount"),
"sql_rowcount": payload.get("sql_rowcount"),
"detected_currency": payload.get("detected_currency"),
+ "semantic_cache_status": payload.get("semantic_cache_status"),
}
return payload
diff --git a/superset/common/query_context_factory.py b/superset/common/query_context_factory.py
index 2a4c66f3d8a..96a1eacde82 100644
--- a/superset/common/query_context_factory.py
+++ b/superset/common/query_context_factory.py
@@ -129,6 +129,7 @@ class QueryContextFactory: # pylint: disable=too-few-public-methods
datasource_model_instance=cast(
"BaseDatasource", datasource_model_instance
),
+ force_query=force,
server_pagination=server_pagination,
preserve_null_row_limit=preserve_null_row_limit,
**query_obj,
diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py
index d7b5bb6e242..0e80ba377e3 100644
--- a/superset/common/query_context_processor.py
+++ b/superset/common/query_context_processor.py
@@ -272,6 +272,12 @@ class QueryContextProcessor:
self._resolve_forced_query(query_obj, cache_key)
or timeout == CACHE_DISABLED_TIMEOUT
)
+ if query_obj:
+ # Datasources that run their own caching (semantic containment)
+ # must honor the same resolved chart/custom timeout and force
+ # decision as the result cache, not the datasource-level default.
+ query_obj.force_query = force_query
+ query_obj.cache_timeout = timeout
query_planning_ns = max(0, time.perf_counter_ns() - query_planning_start_ns)
cache_resolution_start_ns = time.perf_counter_ns()
@@ -416,6 +422,7 @@ class QueryContextProcessor:
"annotation_data": cache.annotation_data,
"error": cache.error_message,
"is_cached": cache.is_cached,
+ "semantic_cache_status": getattr(cache, "semantic_cache_status", None),
"query": cache.query,
"status": cache.status,
"stacktrace": cache.stacktrace,
@@ -679,6 +686,12 @@ class QueryContextProcessor:
return
totals_query = self._query_context.queries[totals_idx]
+ totals_timeout: int = self.get_cache_timeout()
+ totals_query.cache_timeout = totals_timeout
+ totals_query.force_query = (
+ self._resolve_forced_query(totals_query, self.query_cache_key(totals_query))
+ or totals_timeout == CACHE_DISABLED_TIMEOUT
+ )
result = self._query_context.get_query_result(totals_query)
df = result.df
diff --git a/superset/common/query_object.py b/superset/common/query_object.py
index 84809eadf9f..b6dcbe65f62 100644
--- a/superset/common/query_object.py
+++ b/superset/common/query_object.py
@@ -95,10 +95,12 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
annotation_layers: list[dict[str, Any]]
applied_time_extras: dict[str, str]
apply_fetch_values_predicate: bool
+ cache_timeout: int | None
columns: list[Column]
datasource: BaseDatasource | None
extras: dict[str, Any]
filter: list[QueryObjectFilterClause]
+ force_query: bool
from_dttm: datetime | None
granularity: str | None
grouping_sets: list[list[str]]
@@ -128,10 +130,12 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
annotation_layers: list[dict[str, Any]] | None = None,
applied_time_extras: dict[str, str] | None = None,
apply_fetch_values_predicate: bool = False,
+ cache_timeout: int | None = None,
columns: list[Column] | None = None,
datasource: BaseDatasource | None = None,
extras: dict[str, Any] | None = None,
filters: list[QueryObjectFilterClause] | None = None,
+ force_query: bool = False,
granularity: str | None = None,
is_rowcount: bool = False,
is_timeseries: bool | None = None,
@@ -153,10 +157,14 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
self._set_annotation_layers(annotation_layers)
self.applied_time_extras = applied_time_extras or {}
self.apply_fetch_values_predicate = apply_fetch_values_predicate or False
+ # Resolved chart/custom/datasource cache timeout; None until the query
+ # context processor resolves it.
+ self.cache_timeout = cache_timeout
self.columns = columns or []
self.datasource = datasource
self.extras = extras or {}
self.filter = filters or []
+ self.force_query = force_query
self.granularity = granularity
self.is_rowcount = is_rowcount
self._set_is_timeseries(is_timeseries)
@@ -580,9 +588,11 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
def to_dict(self) -> QueryObjectDict:
query_object_dict: QueryObjectDict = {
"apply_fetch_values_predicate": self.apply_fetch_values_predicate,
+ "cache_timeout": self.cache_timeout,
"columns": self.columns,
"extras": self.extras,
"filter": self.filter,
+ "force_query": self.force_query,
"from_dttm": self.from_dttm,
"granularity": self.granularity,
"inner_from_dttm": self.inner_from_dttm,
@@ -642,6 +652,11 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
key=lambda value: (type(value).__name__, str(value)),
)
+ # Force refresh and timeout control cache reads and retention, not the
+ # logical result identity.
+ cache_dict.pop("force_query", None)
+ cache_dict.pop("cache_timeout", None)
+
# TODO: the below KVs can all be cleaned up and moved to `to_dict()` at some
# predetermined point in time when orgs are aware that the previously
# cached results will be invalidated.
diff --git a/superset/common/utils/query_cache_manager.py b/superset/common/utils/query_cache_manager.py
index 2530f8d9770..15f88418963 100644
--- a/superset/common/utils/query_cache_manager.py
+++ b/superset/common/utils/query_cache_manager.py
@@ -25,7 +25,7 @@ from flask_caching import Cache
from pandas import DataFrame
from superset.common.db_query_status import QueryStatus
-from superset.constants import CacheRegion
+from superset.constants import CacheRegion, SemanticCacheStatus
from superset.exceptions import CacheLoadError
from superset.extensions import cache_manager
from superset.models.helpers import QueryResult
@@ -69,6 +69,7 @@ class QueryCacheManager:
cache_value: dict[str, Any] | None = None,
sql_rowcount: int | None = None,
queried_dttm: str | None = None,
+ semantic_cache_status: str | None = None,
) -> None:
self.df = df
self.query = query
@@ -86,6 +87,7 @@ class QueryCacheManager:
self.cache_value = cache_value
self.sql_rowcount = sql_rowcount
self.queried_dttm = queried_dttm
+ self.semantic_cache_status: str | None = semantic_cache_status
self.bq_memory_limited: bool = False
self.bq_memory_limited_row_count: int = 0
# Whether set_query_result actually persisted the value to the backend
@@ -117,6 +119,7 @@ class QueryCacheManager:
self.error_message = query_result.error_message
self.df = query_result.df
self.sql_rowcount = query_result.sql_rowcount
+ self.semantic_cache_status = query_result.semantic_cache_status
self.annotation_data = {} if annotation_data is None else annotation_data
self.queried_dttm = (
datetime.now(tz=timezone.utc).replace(microsecond=0).isoformat()
@@ -147,6 +150,7 @@ class QueryCacheManager:
"rejected_filter_columns": self.rejected_filter_columns,
"annotation_data": self.annotation_data,
"sql_rowcount": self.sql_rowcount,
+ "semantic_cache_status": self.semantic_cache_status,
"queried_dttm": self.queried_dttm,
"dttm": self.queried_dttm, # Backwards compatibility
"bq_memory_limited": self.bq_memory_limited,
@@ -215,6 +219,9 @@ class QueryCacheManager:
query_cache.is_loaded = True
query_cache.is_cached = True
query_cache.sql_rowcount = cache_value.get("sql_rowcount", None)
+ # This request was served by the ordinary result cache. Do not
+ # replay how the cached value was produced by an earlier request.
+ query_cache.semantic_cache_status = SemanticCacheStatus.MISS.value
query_cache.cache_dttm = cache_value["dttm"]
query_cache.queried_dttm = cache_value.get(
"queried_dttm", cache_value.get("dttm")
diff --git a/superset/config.py b/superset/config.py
index 53f86824d2f..a35ed0043f8 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -741,6 +741,12 @@ DEFAULT_FEATURE_FLAGS: dict[str, bool] = {
# Enable semantic layers and show semantic views alongside datasets
# @lifecycle: development
"SEMANTIC_LAYERS": False,
+ # Enable Superset-owned containment caching for opted-in semantic providers.
+ # Requires shared DATA_CACHE_CONFIG and Redis-backed
+ # DISTRIBUTED_COORDINATION_CONFIG.
+ # @docs: https://superset.apache.org/admin-docs/configuration/cache/#semantic-containment-cache
+ # @lifecycle: development
+ "SEMANTIC_LAYER_CONTAINMENT_CACHE": False,
# Enables advanced data type support
# @lifecycle: development
"ENABLE_ADVANCED_DATA_TYPES": False,
@@ -3626,6 +3632,12 @@ DISTRIBUTED_COORDINATION_CONFIG: CacheConfig | None = None
# indefinitely. Defaults to 24 hours.
DISTRIBUTED_COORDINATION_SIGNAL_TTL = int(timedelta(hours=24).total_seconds())
+# Maximum time to wait for a semantic-cache descriptor lease.
+SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS: float = 1.0
+
+# Expiry for owner-token semantic-cache descriptor leases.
+SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS: int = 30
+
# Default lock TTL (time-to-live) in seconds for distributed locks.
# Can be overridden per-call via the `ttl_seconds` parameter.
# After TTL expires, the lock is automatically released to prevent deadlocks.
diff --git a/superset/constants.py b/superset/constants.py
index 42254b0a6f8..db31281aedb 100644
--- a/superset/constants.py
+++ b/superset/constants.py
@@ -18,6 +18,7 @@
# ATTENTION: If you change any constants, make sure to also change utils/common.js
# string to use when None values *need* to be converted to/from strings
+from collections.abc import Iterable
from enum import Enum
from superset.utils.backports import StrEnum
@@ -295,3 +296,41 @@ class CacheRegion(StrEnum):
# Cache timeout constants
CACHE_DISABLED_TIMEOUT = -1 # Special value indicating no caching should occur
+
+
+class SemanticCacheStatus(StrEnum):
+ """Containment-cache provenance of a semantic query result.
+
+ A single chart query can fan out into several semantic queries (a main
+ query plus one per time offset), each served independently. ``HIT`` means
+ every one of them came from containment, ``MISS`` that none did, and
+ ``MIXED`` that provenance differs across them.
+ """
+
+ HIT = "HIT"
+ MISS = "MISS"
+ MIXED = "MIXED"
+
+ @classmethod
+ def from_hits(cls, hits: Iterable[bool]) -> "SemanticCacheStatus":
+ """Roll per-semantic-query hit flags up to one status."""
+ outcomes: tuple[bool, ...] = tuple(hits)
+ if outcomes and all(outcomes):
+ return cls.HIT
+ if any(outcomes):
+ return cls.MIXED
+ return cls.MISS
+
+ @classmethod
+ def combine(cls, statuses: Iterable[object]) -> "SemanticCacheStatus":
+ """Roll per-query statuses up to a response-level status.
+
+ Queries without a status (non-semantic datasources, failures) count as
+ misses so a response is only a ``HIT`` when everything in it was.
+ """
+ values: tuple[object, ...] = tuple(statuses)
+ if values and all(value == cls.HIT for value in values):
+ return cls.HIT
+ if any(value in (cls.HIT, cls.MIXED) for value in values):
+ return cls.MIXED
+ return cls.MISS
diff --git a/superset/coordination/cache_backend.py b/superset/coordination/cache_backend.py
index fa3e9baedf9..c9bb8a4d306 100644
--- a/superset/coordination/cache_backend.py
+++ b/superset/coordination/cache_backend.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+from collections.abc import Callable
from typing import Any
import redis
@@ -35,6 +36,25 @@ else
end
"""
+_COMPARE_AND_EXPIRE_SCRIPT: str = """
+if redis.call('get', KEYS[1]) == ARGV[1] then
+ return redis.call('expire', KEYS[1], ARGV[2])
+end
+return 0
+"""
+
+_COMPARE_AND_SET_LUA: str = """
+if redis.call('get', KEYS[1]) ~= ARGV[1] then
+ return 0
+end
+if tonumber(ARGV[3]) > 0 then
+ redis.call('set', KEYS[2], ARGV[2], 'EX', ARGV[3])
+else
+ redis.call('set', KEYS[2], ARGV[2])
+end
+return 1
+"""
+
class RedisCommandsMixin:
"""Coordination commands issued against the backend's ``redis.Redis`` client.
@@ -105,6 +125,35 @@ class RedisCommandsMixin:
"""
return int(self._cache.eval(_COMPARE_AND_DELETE_LUA, 1, name, expected))
+ def acquire_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ """Acquire an expiring lease only when the key is unowned."""
+ return bool(self._cache.set(key, owner_token, nx=True, ex=lease_seconds))
+
+ def compare_owner_and_set(
+ self,
+ lease_key: str,
+ owner_token: str,
+ key: str,
+ value: bytes,
+ timeout: int,
+ ) -> bool:
+ """Fence a serialized value write on this client's lease atomically."""
+ evaluate: Callable[..., object] = self._cache.eval
+ return bool(
+ evaluate(
+ _COMPARE_AND_SET_LUA, 2, lease_key, key, owner_token, value, timeout
+ )
+ )
+
+ def release_owner_token(self, key: str, owner_token: str) -> bool:
+ """Release the lease only when the caller still owns it."""
+ return bool(self.compare_and_delete(key, owner_token))
+
def publish(self, channel: str, message: str) -> int:
"""
Publish a message to a Redis pub/sub channel.
@@ -183,6 +232,23 @@ class RedisCommandsMixin:
"""Set a TTL (seconds) on a key; used to bound signal-stream growth."""
return bool(self._cache.expire(name, seconds))
+ def refresh_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ """Extend an expiring lease only while the caller still owns it."""
+ return bool(
+ self._cache.eval(
+ _COMPARE_AND_EXPIRE_SCRIPT,
+ 1,
+ key,
+ owner_token,
+ lease_seconds,
+ )
+ )
+
class RedisCacheBackend(RedisCommandsMixin, RedisCache):
def __init__( # pylint: disable=too-many-arguments
@@ -280,12 +346,18 @@ class RedisSentinelCacheBackend(RedisCommandsMixin, RedisSentinelCache):
ssl_ca_certs: str | None = None,
socket_timeout: float | None = None,
socket_connect_timeout: float | None = None,
+ force_master_ip: str | None = None,
**kwargs: Any,
) -> None:
# Sentinel dont directly support SSL
# Initialize Sentinel without SSL parameters
self._sentinel = Sentinel(
sentinels,
+ **(
+ {"force_master_ip": force_master_ip}
+ if force_master_ip is not None
+ else {}
+ ),
# See the matching comment in RedisCacheBackend.__init__: pin the
# pre-redis-py-8 defaults (no socket timeout, RESP2) explicitly
# for the sentinel-node connections too, so this bump doesn't
diff --git a/superset/initialization/__init__.py b/superset/initialization/__init__.py
index 93111e9b586..af05d3fb5b0 100644
--- a/superset/initialization/__init__.py
+++ b/superset/initialization/__init__.py
@@ -1083,6 +1083,7 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
self.configure_auth_provider()
self.configure_ssh_manager()
self.configure_stats_manager()
+ self.configure_semantic_cache()
self.configure_task_manager()
self.configure_websocket()
@@ -1453,6 +1454,93 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
def configure_stats_manager(self) -> None:
stats_logger_manager.init_app(self.superset_app)
+ def configure_semantic_cache(self) -> None:
+ """Configure optional semantic containment caching for this process."""
+ from typing import cast
+
+ from superset.semantic_layers.cache import (
+ initialize_semantic_cache,
+ SemanticCacheDisabledReason,
+ SemanticCacheMetrics,
+ SemanticCacheState,
+ )
+ from superset.semantic_layers.cache_repository import SemanticCacheBackend
+
+ requested: bool = feature_flag_manager.is_feature_enabled(
+ "SEMANTIC_LAYER_CONTAINMENT_CACHE"
+ )
+ state: SemanticCacheState = initialize_semantic_cache(
+ parent_enabled=feature_flag_manager.is_feature_enabled("SEMANTIC_LAYERS"),
+ requested=requested,
+ backend=cast(SemanticCacheBackend, cache_manager.data_cache),
+ coordination=cache_manager.semantic_cache_coordination,
+ wait_seconds=self.config["SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS"],
+ lease_seconds=self.config["SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS"],
+ metrics=cast(SemanticCacheMetrics, stats_logger_manager.instance),
+ # The same cap the ordinary result cache applies to a single value;
+ # containment values share that backend.
+ max_value_bytes=self.config.get("DATA_CACHE_MAX_VALUE_SIZE"),
+ )
+ try:
+ stats_logger_manager.instance.gauge(
+ "semantic_cache.containment.enabled",
+ float(state.effective),
+ )
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug("Semantic cache startup metric emission failed", exc_info=True)
+ if (
+ state.disabled_reason
+ is SemanticCacheDisabledReason.UNSUPPORTED_COORDINATION
+ ):
+ try:
+ stats_logger_manager.instance.incr(
+ "semantic_cache.containment.unsupported"
+ )
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug(
+ "Semantic cache startup metric emission failed",
+ exc_info=True,
+ )
+ logger.warning(
+ "Semantic containment caching was requested but the configured "
+ "distributed coordination backend does not support owner-token leases; "
+ "containment caching is disabled for this process"
+ )
+ elif state.disabled_reason is SemanticCacheDisabledReason.UNSUPPORTED_BACKEND:
+ try:
+ stats_logger_manager.instance.incr(
+ "semantic_cache.containment.unsupported"
+ )
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug(
+ "Semantic cache startup metric emission failed",
+ exc_info=True,
+ )
+ logger.warning(
+ "Semantic containment caching was requested but DATA_CACHE_CONFIG "
+ "is not a persistent shared cache that containment can read back "
+ "from (NullCache never retains values; RedisSentinelCache reads "
+ "from replicas); containment caching is disabled for this process"
+ )
+ elif (
+ state.disabled_reason
+ is SemanticCacheDisabledReason.INVALID_COORDINATION_CONFIGURATION
+ ):
+ try:
+ stats_logger_manager.instance.incr(
+ "semantic_cache.containment.invalid_configuration"
+ )
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug(
+ "Semantic cache startup metric emission failed",
+ exc_info=True,
+ )
+ logger.warning(
+ "Semantic containment caching was requested but its coordination "
+ "timing configuration is invalid; containment caching is disabled "
+ "for this process"
+ )
+
def setup_event_logger(self) -> None:
_event_logger["event_logger"] = get_event_logger_from_cfg_value(
self.superset_app.config.get("EVENT_LOGGER", DBEventLogger())
diff --git a/superset/models/helpers.py b/superset/models/helpers.py
index e3ce56ba5ba..77220ccd28f 100644
--- a/superset/models/helpers.py
+++ b/superset/models/helpers.py
@@ -1609,6 +1609,7 @@ class QueryResult: # pylint: disable=too-few-public-methods
from_dttm: Optional[datetime] = None,
to_dttm: Optional[datetime] = None,
sql_shifted_temporal_labels: set[str] | None = None,
+ semantic_cache_status: Optional[str] = None,
) -> None:
self.df = df
self.query = query
@@ -1622,6 +1623,8 @@ class QueryResult: # pylint: disable=too-few-public-methods
self.from_dttm = from_dttm
self.to_dttm = to_dttm
self.sql_shifted_temporal_labels = sql_shifted_temporal_labels or set()
+ # A ``SemanticCacheStatus`` value, or None for non-semantic sources.
+ self.semantic_cache_status: Optional[str] = semantic_cache_status
self.sql_rowcount = len(self.df.index) if not self.df.empty else 0
diff --git a/superset/semantic_layers/cache.py b/superset/semantic_layers/cache.py
new file mode 100644
index 00000000000..01b2c345451
--- /dev/null
+++ b/superset/semantic_layers/cache.py
@@ -0,0 +1,496 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Application-level state for semantic containment caching."""
+
+from __future__ import annotations
+
+import enum
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from flask import current_app, has_app_context
+from flask_caching.backends.nullcache import NullCache
+from flask_caching.backends.rediscache import RedisCache, RedisSentinelCache
+from redis.backoff import NoBackoff
+from redis.retry import Retry
+from superset_core.semantic_layers.types import SemanticQuery, SemanticResult
+
+from superset.coordination.cache_backend import (
+ RedisCacheBackend,
+ RedisCommandsMixin,
+ RedisSentinelCacheBackend,
+)
+from superset.semantic_layers.cache_coordination import (
+ OwnerTokenCoordinationBackend,
+ SemanticCacheCoordinationSettings,
+ SemanticCacheCoordinator,
+)
+from superset.semantic_layers.cache_policy import (
+ ContainmentCapabilities,
+ eligible_reuse_mode as eligible_reuse_mode,
+ rank_eligible_entries as rank_eligible_entries,
+)
+from superset.semantic_layers.cache_repository import (
+ SemanticCacheBackend,
+ SemanticCacheBackendError,
+ SemanticCacheLookupError,
+ SemanticCacheLookupResult,
+ SemanticCacheRepository,
+ SemanticCacheStoreError,
+ SemanticCacheWriteFence,
+ ViewMeta,
+)
+from superset.semantic_layers.cache_transform import (
+ SemanticCacheTransformationError,
+ transform_result,
+)
+
+
+class SemanticCacheDisabledReason(str, enum.Enum):
+ """Stable operational reasons why containment is ineffective."""
+
+ PARENT_FLAG_OFF = "parent_flag_off"
+ FLAG_OFF = "flag_off"
+ UNSUPPORTED_COORDINATION = "unsupported_coordination"
+ INVALID_COORDINATION_CONFIGURATION = "invalid_coordination_configuration"
+ UNSUPPORTED_BACKEND = "unsupported_backend"
+
+
+@dataclass(frozen=True)
+class SemanticCacheState:
+ """Requested and effective containment state owned by the cache service."""
+
+ requested: bool
+ effective: bool
+ disabled_reason: SemanticCacheDisabledReason | None
+
+ def __post_init__(self) -> None:
+ if self.effective and not self.requested:
+ raise ValueError("Effective semantic caching must be requested")
+ if self.effective and self.disabled_reason is not None:
+ raise ValueError("Effective semantic caching cannot have a disabled reason")
+ if not self.effective and self.disabled_reason is None:
+ raise ValueError("Ineffective semantic caching requires a disabled reason")
+
+ @classmethod
+ def enabled(cls) -> SemanticCacheState:
+ """Return the single valid effective state."""
+ return cls(requested=True, effective=True, disabled_reason=None)
+
+ @classmethod
+ def disabled(
+ cls,
+ reason: SemanticCacheDisabledReason,
+ *,
+ requested: bool = False,
+ ) -> SemanticCacheState:
+ """Return an explicitly ineffective state."""
+ return cls(
+ requested=requested,
+ effective=False,
+ disabled_reason=reason,
+ )
+
+
+@dataclass(frozen=True)
+class SemanticCacheOutcome:
+ """Host-side result plus whether containment supplied it."""
+
+ result: SemanticResult
+ cache_hit: bool
+
+
+class SemanticCacheMetrics(Protocol):
+ """Fixed-name metrics used by containment cache operations."""
+
+ def incr(self, key: str) -> None: ... # pragma: no cover
+
+ def gauge(self, key: str, value: float) -> None: ... # pragma: no cover
+
+
+SEMANTIC_CACHE_METRIC_PREFIX: str = "semantic_cache.containment"
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class SafeSemanticCacheBackend:
+ """Translate pluggable cache-client failures into the repository boundary."""
+
+ def __init__(self, backend: SemanticCacheBackend) -> None:
+ self._backend: SemanticCacheBackend = backend
+
+ def get(self, key: str) -> object | None:
+ try:
+ return self._backend.get(key)
+ except Exception as ex: # pylint: disable=broad-exception-caught
+ raise SemanticCacheBackendError("Semantic cache get failed") from ex
+
+ def has(self, key: str) -> bool:
+ try:
+ return bool(self._backend.has(key))
+ except Exception as ex: # pylint: disable=broad-exception-caught
+ raise SemanticCacheBackendError("Semantic cache has failed") from ex
+
+ def set(
+ self,
+ key: str,
+ value: object,
+ timeout: int | None = None,
+ ) -> bool:
+ try:
+ return self._backend.set(key, value, timeout=timeout)
+ except Exception as ex: # pylint: disable=broad-exception-caught
+ raise SemanticCacheBackendError("Semantic cache set failed") from ex
+
+ def delete(self, key: str) -> bool:
+ try:
+ return self._backend.delete(key)
+ except Exception as ex: # pylint: disable=broad-exception-caught
+ raise SemanticCacheBackendError("Semantic cache delete failed") from ex
+
+ def set_if_owner(
+ self,
+ fence: SemanticCacheWriteFence,
+ key: str,
+ value: object,
+ timeout: int | None,
+ ) -> bool | None:
+ """Use Lua only when the value and lease share the exact Redis client."""
+ inner: Any = getattr(self._backend, "cache", self._backend)
+ coordination: RedisCommandsMixin | None = fence.redis_backend
+ # DATA_CACHE_CONFIG and DISTRIBUTED_COORDINATION_CONFIG construct
+ # separate clients (possibly separate databases/servers). Never run a
+ # cross-client Lua fence: None selects the non-atomic immediate recheck.
+ if (
+ not isinstance(inner, RedisCache)
+ or coordination is None
+ or inner._write_client is not coordination._cache
+ ):
+ return None
+ try:
+ return coordination.compare_owner_and_set(
+ fence.key,
+ fence.owner_token,
+ f"{inner._get_prefix()}{key}",
+ inner.serializer.dumps(value),
+ inner._normalize_timeout(timeout),
+ )
+ except Exception as ex: # pylint: disable=broad-exception-caught
+ raise SemanticCacheBackendError("Semantic cache fenced set failed") from ex
+
+
+def _bounded_coordination_backend(
+ backend: OwnerTokenCoordinationBackend,
+ wait_seconds: float,
+) -> OwnerTokenCoordinationBackend:
+ """Construct a private, bounded Redis client from the coordination config."""
+ if not isinstance(backend, (RedisCacheBackend, RedisSentinelCacheBackend)):
+ return backend
+ if not has_app_context():
+ # Explicitly injected clients outside app initialization remain owned
+ # by their caller, including their I/O timeout configuration.
+ return backend
+ config: dict[str, Any] = dict(current_app.config["DISTRIBUTED_COORDINATION_CONFIG"])
+ # WAIT_SECONDS bounds contention, not network calls. Derive a positive
+ # per-I/O bound too; zero wait means no contention retries, not infinite I/O.
+ io_timeout: float = max(0.001, wait_seconds)
+ config["CACHE_REDIS_SOCKET_TIMEOUT"] = io_timeout
+ config["CACHE_REDIS_SOCKET_CONNECT_TIMEOUT"] = io_timeout
+ bounded: RedisCacheBackend | RedisSentinelCacheBackend = type(backend).from_config(
+ config
+ )
+ clients: list[Any] = [bounded._cache]
+ if isinstance(bounded, RedisSentinelCacheBackend):
+ clients.extend(bounded._sentinel.sentinels)
+ for client in clients:
+ # Disable redis-py's retry/backoff budget on these private connections.
+ client.connection_pool.connection_kwargs["retry"] = Retry(NoBackoff(), 0)
+ return bounded
+
+
+class SemanticCacheService:
+ """Orchestrate optional cache lookup, provider execution, and storage.
+
+ This service is intentionally limited to application flow. Cache identity,
+ containment proofs, result transformation, persistence, distributed mutation,
+ and Superset host adaptation live in ``cache_identity``, ``cache_policy``,
+ ``cache_transform``, ``cache_repository``, ``cache_coordination``, and
+ ``cache_host`` respectively. Keeping those decisions behind narrow boundaries
+ prevents the orchestration service from becoming a multipurpose cache class.
+ """
+
+ def __init__(
+ self,
+ state: SemanticCacheState,
+ repository: SemanticCacheRepository | None = None,
+ metrics: SemanticCacheMetrics | None = None,
+ *,
+ max_value_bytes: int | None = None,
+ ) -> None:
+ if state.effective and repository is None:
+ raise ValueError("Effective semantic caching requires a repository")
+ self.state: SemanticCacheState = state
+ self._repository: SemanticCacheRepository | None = repository
+ self._metrics: SemanticCacheMetrics | None = metrics
+ # Results above this many Arrow bytes are served but never stored.
+ # The value is pickled inside the lease, where a large payload stalls
+ # renewal and can flood the shared data cache; None disables the cap.
+ self._max_value_bytes: int | None = max_value_bytes
+
+ def _increment(self, suffix: str) -> None:
+ metrics: SemanticCacheMetrics | None = self._metrics
+ if metrics is None:
+ return
+ try:
+ metrics.incr(f"{SEMANTIC_CACHE_METRIC_PREFIX}.{suffix}")
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug("Semantic cache metric emission failed", exc_info=True)
+
+ @classmethod
+ def default_ineffective(cls) -> SemanticCacheService:
+ """Build the safe process default used before capability initialization."""
+ return cls(SemanticCacheState.disabled(SemanticCacheDisabledReason.FLAG_OFF))
+
+ def execute(
+ self,
+ meta: ViewMeta,
+ query: SemanticQuery,
+ provider: Callable[[SemanticQuery], SemanticResult],
+ *,
+ capabilities: ContainmentCapabilities,
+ force: bool = False,
+ ) -> SemanticCacheOutcome:
+ """Serve a proven hit or execute and best-effort store provider output."""
+ repository: SemanticCacheRepository | None = self._repository
+ if not self.state.effective or repository is None:
+ self._increment("bypass")
+ return SemanticCacheOutcome(provider(query), cache_hit=False)
+ # Every cache stage degrades on ANY exception rather than failing the
+ # request: cached material outlives the code that wrote it, so a
+ # deploy that reshapes a pickled entry (or any unanticipated backend
+ # fault) must cost a miss or a skipped store, never the user's chart.
+ if not force:
+ served: SemanticCacheOutcome | None
+ proceed_to_store: bool
+ served, proceed_to_store = self._serve_from_cache(
+ repository, meta, query, capabilities
+ )
+ if served is not None:
+ return served
+ if not proceed_to_store:
+ return SemanticCacheOutcome(provider(query), cache_hit=False)
+ else:
+ self._increment("bypass")
+ result: SemanticResult = provider(query)
+ if self._exceeds_size_cap(result):
+ self._increment("store_skipped")
+ return SemanticCacheOutcome(result, cache_hit=False)
+ try:
+ # A forced refresh must replace whatever is cached; an ordinary
+ # miss may find that a concurrent identical request already stored
+ # the same value, in which case the store is redundant.
+ repository.store(meta, query, result, replace=force)
+ except SemanticCacheStoreError:
+ self._increment("store_failure")
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.warning("Semantic cache store failed unexpectedly", exc_info=True)
+ self._increment("store_failure")
+ return SemanticCacheOutcome(result, cache_hit=False)
+
+ def _exceeds_size_cap(self, result: SemanticResult) -> bool:
+ if self._max_value_bytes is None or result.results is None:
+ return False
+ return result.results.nbytes > self._max_value_bytes
+
+ def _serve_from_cache(
+ self,
+ repository: SemanticCacheRepository,
+ meta: ViewMeta,
+ query: SemanticQuery,
+ capabilities: ContainmentCapabilities,
+ ) -> tuple[SemanticCacheOutcome | None, bool]:
+ """Attempt a cache read; return (hit outcome or None, store on miss).
+
+ The second element is False when the read phase failed outright — the
+ caller should execute the provider without attempting a store, since
+ the backend just demonstrated it is unhealthy.
+ """
+ try:
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ meta, query, capabilities
+ )
+ except SemanticCacheLookupError:
+ self._increment("lookup_failure")
+ return None, False
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.warning("Semantic cache lookup failed unexpectedly", exc_info=True)
+ self._increment("lookup_failure")
+ return None, False
+ if lookup_result.fence_rejections:
+ self._increment("fence_reject")
+ for candidate in lookup_result.candidates:
+ try:
+ transformed: SemanticResult = transform_result(
+ candidate.result,
+ query,
+ candidate.decision,
+ capabilities,
+ )
+ except SemanticCacheTransformationError:
+ self._increment("transform_failure")
+ continue
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.warning(
+ "Semantic cache transformation failed unexpectedly",
+ exc_info=True,
+ )
+ self._increment("transform_failure")
+ continue
+ self._increment("hit")
+ return SemanticCacheOutcome(transformed, cache_hit=True), False
+ try:
+ repository.prune_missing(
+ meta,
+ lookup_result.missing_value_keys,
+ )
+ except SemanticCacheLookupError:
+ self._increment("prune_failure")
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.warning("Semantic cache pruning failed unexpectedly", exc_info=True)
+ self._increment("prune_failure")
+ self._increment("miss")
+ return None, True
+
+ def execute_provider(
+ self,
+ query: SemanticQuery,
+ provider: Callable[[SemanticQuery], SemanticResult],
+ ) -> SemanticCacheOutcome:
+ """Execute a provider while recording an intentional cache bypass."""
+ self._increment("bypass")
+ return SemanticCacheOutcome(provider(query), cache_hit=False)
+
+
+def _backend_reads_from_replicas(backend: object) -> bool:
+ """Detect cache backends whose reads may lag their own writes."""
+ inner: object = getattr(backend, "cache", backend)
+ return isinstance(backend, RedisSentinelCache) or isinstance(
+ inner, RedisSentinelCache
+ )
+
+
+def _backend_discards_values(backend: object) -> bool:
+ """Detect cache backends that never retain a stored value."""
+ inner: object = getattr(backend, "cache", backend)
+ return isinstance(backend, NullCache) or isinstance(inner, NullCache)
+
+
+def initialize_semantic_cache(
+ *,
+ parent_enabled: bool,
+ requested: bool,
+ backend: SemanticCacheBackend,
+ coordination: OwnerTokenCoordinationBackend | None,
+ wait_seconds: float,
+ lease_seconds: int,
+ metrics: SemanticCacheMetrics | None = None,
+ max_value_bytes: int | None = None,
+) -> SemanticCacheState:
+ """Replace process cache state from validated startup configuration."""
+ global semantic_cache_service
+
+ if not parent_enabled:
+ state: SemanticCacheState = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.PARENT_FLAG_OFF,
+ requested=requested,
+ )
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+ if not requested:
+ state = SemanticCacheState.disabled(SemanticCacheDisabledReason.FLAG_OFF)
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+ if _backend_discards_values(backend):
+ # Containment values and descriptors live in the data cache. The
+ # default ``NullCache`` accepts every store and answers every lookup
+ # with a miss, so the feature would look enabled while paying for
+ # coordination on every request and never serving a hit. Fail closed
+ # so the operator sees the misconfiguration at startup.
+ state = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.UNSUPPORTED_BACKEND,
+ requested=True,
+ )
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+ if _backend_reads_from_replicas(backend):
+ # Sentinel-backed caches read through ``slave_for`` replicas while
+ # writing to the master, so the lease-guarded read-modify-write on
+ # descriptor buckets loses read-your-writes under replication lag:
+ # a freshly stored descriptor can be invisible to the next lease
+ # holder, which then persists the stale bucket over it. Fail closed
+ # until repository reads are routed through the write client.
+ state = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.UNSUPPORTED_BACKEND,
+ requested=True,
+ )
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+ if coordination is None:
+ state = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.UNSUPPORTED_COORDINATION,
+ requested=True,
+ )
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+ try:
+ settings: SemanticCacheCoordinationSettings = SemanticCacheCoordinationSettings(
+ wait_seconds, lease_seconds
+ )
+ except (TypeError, ValueError):
+ state = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.INVALID_COORDINATION_CONFIGURATION,
+ requested=True,
+ )
+ semantic_cache_service = SemanticCacheService(state, metrics=metrics)
+ return state
+
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ _bounded_coordination_backend(coordination, settings.wait_seconds),
+ settings,
+ failure_metric=(
+ (lambda key: metrics.incr(key)) if metrics is not None else None
+ ),
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ SafeSemanticCacheBackend(backend),
+ coordinator,
+ )
+ state = SemanticCacheState.enabled()
+ semantic_cache_service = SemanticCacheService(
+ state,
+ repository,
+ metrics,
+ max_value_bytes=max_value_bytes,
+ )
+ return state
+
+
+semantic_cache_service: SemanticCacheService = (
+ SemanticCacheService.default_ineffective()
+)
diff --git a/superset/semantic_layers/cache_coordination.py b/superset/semantic_layers/cache_coordination.py
new file mode 100644
index 00000000000..303a1cecca4
--- /dev/null
+++ b/superset/semantic_layers/cache_coordination.py
@@ -0,0 +1,240 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Ownership-safe coordination for semantic-cache descriptor mutations."""
+
+import logging
+import math
+from collections.abc import Callable
+from contextvars import Token
+from dataclasses import dataclass
+from random import random
+from threading import Event, Thread
+from time import monotonic, sleep
+from typing import Protocol, runtime_checkable
+from uuid import uuid4
+
+from redis.exceptions import RedisError
+
+from superset.coordination.cache_backend import RedisCommandsMixin
+from superset.semantic_layers.cache_repository import (
+ semantic_cache_write_fence,
+ SemanticCacheCoordinationError,
+ SemanticCacheWriteFence,
+)
+
+SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC: str = (
+ "semantic_cache.containment.coordination_failure"
+)
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@runtime_checkable
+class OwnerTokenCoordinationBackend(Protocol):
+ """Atomic lease operations required from a coordination backend."""
+
+ def acquire_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool: ... # pragma: no cover
+
+ def release_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ ) -> bool: ... # pragma: no cover
+
+ def refresh_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool: ... # pragma: no cover
+
+
+@dataclass(frozen=True)
+class SemanticCacheCoordinationSettings:
+ """Validated bounded timing settings for descriptor leases."""
+
+ wait_seconds: float
+ lease_seconds: int
+
+ def __post_init__(self) -> None:
+ if (
+ isinstance(self.wait_seconds, bool)
+ or not isinstance(self.wait_seconds, (int, float))
+ or not math.isfinite(self.wait_seconds)
+ or self.wait_seconds < 0
+ ):
+ raise ValueError(
+ "Coordination wait seconds must be finite and non-negative"
+ )
+ if (
+ isinstance(self.lease_seconds, bool)
+ or not isinstance(self.lease_seconds, int)
+ or self.lease_seconds <= 0
+ ):
+ raise ValueError("Coordination lease seconds must be positive")
+
+
+class SemanticCacheCoordinator:
+ """Run descriptor mutations while holding an owner-token lease."""
+
+ def __init__(
+ self,
+ backend: OwnerTokenCoordinationBackend,
+ settings: SemanticCacheCoordinationSettings,
+ *,
+ clock: Callable[[], float] = monotonic,
+ sleeper: Callable[[float], None] = sleep,
+ token_factory: Callable[[], str] = lambda: uuid4().hex,
+ failure_metric: Callable[[str], None] | None = None,
+ jitter: Callable[[], float] = random,
+ ) -> None:
+ self._backend: OwnerTokenCoordinationBackend = backend
+ self._settings: SemanticCacheCoordinationSettings = settings
+ self._clock: Callable[[], float] = clock
+ self._sleeper: Callable[[float], None] = sleeper
+ self._token_factory: Callable[[], str] = token_factory
+ self._failure_metric: Callable[[str], None] = failure_metric or (lambda _: None)
+ self._jitter: Callable[[], float] = jitter
+
+ def _failure(self, message: str, cause: RedisError | None = None) -> None:
+ self._record_failure()
+ error: SemanticCacheCoordinationError = SemanticCacheCoordinationError(message)
+ if cause is None:
+ raise error
+ raise error from cause
+
+ def _record_failure(self) -> None:
+ try:
+ self._failure_metric(SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC)
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.debug("Semantic cache coordination metric failed", exc_info=True)
+
+ def _run_with_renewal(
+ self,
+ lease_key: str,
+ owner_token: str,
+ operation: Callable[[], None],
+ ) -> bool:
+ renewal_stopped: Event = Event()
+ renewal_failed: Event = Event()
+
+ def renew_lease() -> None:
+ interval: float = self._settings.lease_seconds / 3
+ while not renewal_stopped.wait(interval):
+ try:
+ refreshed: bool = self._backend.refresh_owner_token(
+ lease_key,
+ owner_token,
+ self._settings.lease_seconds,
+ )
+ except RedisError:
+ renewal_failed.set()
+ return
+ if not refreshed:
+ renewal_failed.set()
+ return
+
+ renewal_thread: Thread = Thread(
+ target=renew_lease,
+ name="semantic-cache-lease-renewal",
+ daemon=True,
+ )
+ renewal_thread.start()
+ try:
+ operation()
+ except Exception: # pylint: disable=broad-exception-caught
+ logger.exception("Semantic cache mutation failed before lease release")
+ raise
+ finally:
+ renewal_stopped.set()
+ renewal_thread.join()
+ return renewal_failed.is_set()
+
+ def mutate(self, key: str, operation: Callable[[], None]) -> bool:
+ """Acquire a bounded lease, mutate, and release only our ownership."""
+ lease_key: str = f"semantic-cache-lock:{key}"
+ owner_token: str = self._token_factory()
+ deadline: float = self._clock() + self._settings.wait_seconds
+ acquired: bool = False
+ while True:
+ try:
+ acquired = self._backend.acquire_owner_token(
+ lease_key,
+ owner_token,
+ self._settings.lease_seconds,
+ )
+ except RedisError as ex:
+ self._failure("Semantic cache lease acquisition failed", ex)
+ if acquired:
+ break
+ remaining: float = deadline - self._clock()
+ if remaining <= 0:
+ self._record_failure()
+ return False
+ # Jitter the spin so a herd of waiters released by one lease
+ # expiry does not retry in lockstep; the wait stays within 50 ms.
+ self._sleeper(min(0.025 + 0.025 * self._jitter(), remaining))
+
+ renewal_failed: bool
+
+ def check_owner() -> bool:
+ """Check the caller's lease immediately before a fallback write."""
+ try:
+ return self._backend.refresh_owner_token(
+ lease_key, owner_token, self._settings.lease_seconds
+ )
+ except RedisError as ex:
+ self._failure("Semantic cache lease check failed", ex)
+ return False # pragma: no cover
+
+ fence_token: Token[SemanticCacheWriteFence | None] = (
+ semantic_cache_write_fence.set(
+ SemanticCacheWriteFence(
+ lease_key,
+ owner_token,
+ check_owner,
+ self._backend
+ if isinstance(self._backend, RedisCommandsMixin)
+ else None,
+ )
+ )
+ )
+ try:
+ renewal_failed = self._run_with_renewal(
+ lease_key,
+ owner_token,
+ operation,
+ )
+ finally:
+ semantic_cache_write_fence.reset(fence_token)
+ try:
+ released: bool = self._backend.release_owner_token(
+ lease_key,
+ owner_token,
+ )
+ except RedisError as ex:
+ self._failure("Semantic cache lease release failed", ex)
+ if not released:
+ self._failure("Semantic cache lease ownership was lost")
+ if renewal_failed:
+ self._failure("Semantic cache lease renewal failed")
+ return True
diff --git a/superset/semantic_layers/cache_host.py b/superset/semantic_layers/cache_host.py
new file mode 100644
index 00000000000..067be38473d
--- /dev/null
+++ b/superset/semantic_layers/cache_host.py
@@ -0,0 +1,248 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Host adaptation for semantic cache identity and provider contracts."""
+
+import hashlib
+import logging
+from collections.abc import Iterable, Mapping
+from datetime import datetime
+from typing import cast, Protocol
+
+from flask import g, has_request_context
+from superset_core.semantic_layers.layer import (
+ SemanticCacheCapabilities,
+ SemanticCacheExecutionContext,
+ SemanticCacheIdentityMaterial,
+ SemanticCacheResponsibility,
+ SemanticCacheScope,
+)
+
+from superset import security_manager
+from superset.connectors.sqla.models import BaseDatasource
+from superset.constants import CACHE_DISABLED_TIMEOUT
+from superset.semantic_layers.cache_identity import (
+ SemanticCacheIdentityFactory,
+ SemanticViewIdentity,
+ SensitiveIdentityMaterialError,
+)
+from superset.semantic_layers.cache_policy import (
+ ContainmentCapabilities,
+ PatternSemantics,
+)
+from superset.semantic_layers.cache_repository import ViewMeta
+from superset.utils import json
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class _SemanticCacheProvider(Protocol):
+ semantic_cache_responsibility: SemanticCacheResponsibility
+ semantic_cache_scope: SemanticCacheScope
+ semantic_cache_capabilities: SemanticCacheCapabilities
+
+ def get_semantic_cache_provider_identity(
+ self,
+ ) -> SemanticCacheIdentityMaterial | None: ... # pragma: no cover
+
+ def get_semantic_cache_context_identity(
+ self,
+ context: SemanticCacheExecutionContext,
+ ) -> SemanticCacheIdentityMaterial | None: ... # pragma: no cover
+
+
+def _execution_context(
+ datasource: BaseDatasource,
+) -> SemanticCacheExecutionContext | None:
+ if not has_request_context() or not getattr(g, "user", None):
+ return None
+ user: object = g.user
+ principal: object = getattr(user, "id", None) or getattr(user, "username", None)
+ if principal is None:
+ return None
+ roles_value: object = getattr(user, "roles", ())
+ roles: Iterable[object] = (
+ roles_value
+ if isinstance(roles_value, Iterable) and not isinstance(roles_value, str)
+ else ()
+ )
+ role_ids: tuple[str, ...] = tuple(
+ sorted(
+ str(getattr(role, "id", None) or getattr(role, "name", ""))
+ for role in roles
+ )
+ )
+ guest_token: object = getattr(user, "guest_token", None)
+ rls_cache_key: list[str] = security_manager.get_rls_cache_key(datasource)
+ identity_payload: str = json.dumps(
+ {
+ "guest_token": guest_token,
+ "principal": str(principal),
+ "rls": rls_cache_key,
+ "roles": role_ids,
+ },
+ default=str,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
+ host_identity: str = hashlib.sha256(identity_payload.encode()).hexdigest()
+ return SemanticCacheExecutionContext(str(principal), role_ids, host_identity)
+
+
+def build_cache_configuration(
+ datasource: BaseDatasource,
+ *,
+ cache_timeout: int | None = None,
+) -> tuple[ViewMeta, ContainmentCapabilities] | None:
+ """Build safe host metadata when the provider explicitly opts in.
+
+ :param cache_timeout: the timeout the query context resolved for this
+ request (custom, then chart, then datasource). Containment must
+ follow the same decision as the ordinary result cache, so it takes
+ precedence over ``datasource.cache_timeout`` whenever it is known.
+ """
+ timeout: int | None = (
+ datasource.cache_timeout if cache_timeout is None else cache_timeout
+ )
+ if timeout == CACHE_DISABLED_TIMEOUT:
+ # Caching is explicitly disabled for this request. Passing the
+ # sentinel to a cache backend would invert it (Redis reads a negative
+ # TTL as "no expiry"), so containment caching is bypassed entirely
+ # instead: nothing is read and nothing is stored.
+ return None
+ token: str | None = _metadata_cache_token(datasource)
+ if token is None:
+ # The host generation is required; never reuse a legacy definition key.
+ return None
+ discriminator: tuple[str, str] | None = cast(
+ tuple[str, str] | None,
+ getattr(datasource, "result_cache_discriminator", None),
+ )
+ layer: _SemanticCacheProvider = cast(
+ _SemanticCacheProvider,
+ datasource.semantic_layer.implementation,
+ )
+ if (
+ getattr(layer, "semantic_cache_responsibility", None)
+ is not SemanticCacheResponsibility.SUPERSET
+ ):
+ return None
+ provider_material: object = layer.get_semantic_cache_provider_identity()
+ if not isinstance(provider_material, SemanticCacheIdentityMaterial):
+ return None
+ scope_material: Mapping[str, object] | None = _scope_material(datasource, layer)
+ if scope_material is None:
+ return None
+ provider_capabilities: object = getattr(layer, "semantic_cache_capabilities", None)
+ if not isinstance(provider_capabilities, SemanticCacheCapabilities):
+ return None
+ pattern_semantics: PatternSemantics | None = (
+ PatternSemantics.sql_like(escape=provider_capabilities.pattern_escape)
+ if provider_capabilities.pattern_escape is not None
+ else None
+ )
+ capabilities: ContainmentCapabilities = ContainmentCapabilities(
+ comparisons=provider_capabilities.comparisons,
+ membership=provider_capabilities.membership,
+ nulls=provider_capabilities.nulls,
+ pattern_semantics=pattern_semantics,
+ )
+ changed_on: object = getattr(datasource, "changed_on", None)
+ definition_material: dict[str, object] = {
+ "changed_on": changed_on.isoformat()
+ if isinstance(changed_on, datetime)
+ else str(changed_on),
+ }
+ # The captured host generation changes on metadata clear, not changed_on.
+ # It is an identity, not a secret; "token" keys trip the secret-material guard.
+ definition_material["metadata_observation"] = token
+ if discriminator is not None:
+ definition_material["result_version"] = (
+ "semantic-result-version",
+ *discriminator,
+ )
+ meta: ViewMeta | None = _view_meta(
+ datasource, definition_material, provider_material, scope_material, timeout
+ )
+ if meta is None:
+ return None
+ return meta, capabilities
+
+
+def _metadata_cache_token(datasource: BaseDatasource) -> str | None:
+ """Return the captured ORM host generation, never a provider-only token."""
+ token: object = getattr(datasource, "metadata_cache_token", None)
+ return token if isinstance(token, str) and token else None
+
+
+def _scope_material(
+ datasource: BaseDatasource,
+ layer: _SemanticCacheProvider,
+) -> Mapping[str, object] | None:
+ """Resolve the reuse scope's identity material, or None to bypass."""
+ scope: object = getattr(layer, "semantic_cache_scope", None)
+ if scope is SemanticCacheScope.GLOBAL:
+ if security_manager.get_rls_cache_key(datasource):
+ # Global reuse rests on the provider's guarantee that results do
+ # not vary by principal; Superset-side row-level security is a
+ # per-principal variation the provider cannot see, so it wins.
+ return None
+ return {"scope": "global"}
+ if scope is SemanticCacheScope.EXECUTION_CONTEXT:
+ context: SemanticCacheExecutionContext | None = _execution_context(datasource)
+ if context is None:
+ return None
+ context_material: object = layer.get_semantic_cache_context_identity(context)
+ if not isinstance(context_material, SemanticCacheIdentityMaterial):
+ return None
+ return {
+ "host_identity": context.host_identity,
+ "provider_identity": context_material.values,
+ }
+ return None
+
+
+def _view_meta(
+ datasource: BaseDatasource,
+ definition_material: Mapping[str, object],
+ provider_material: SemanticCacheIdentityMaterial,
+ scope_material: Mapping[str, object],
+ timeout: int | None,
+) -> ViewMeta | None:
+ """Digest the identity material, or bypass when a provider offers secrets."""
+ try:
+ return ViewMeta(
+ view_identity=SemanticViewIdentity(str(datasource.uuid)),
+ definition_identity=SemanticCacheIdentityFactory.definition(
+ definition_material
+ ),
+ provider_identity=SemanticCacheIdentityFactory.provider(
+ provider_material.values
+ ),
+ scope_identity=SemanticCacheIdentityFactory.scope(scope_material),
+ timeout=timeout,
+ )
+ except SensitiveIdentityMaterialError:
+ # Identity material is the provider's contract to get right; a
+ # secret-like key must never turn into a failed chart. Bypass
+ # containment for the view and say why, without echoing the material.
+ logger.warning(
+ "Semantic containment caching bypassed for view %s: the provider "
+ "offered secret-like identity material",
+ datasource.uuid,
+ )
+ return None
diff --git a/superset/semantic_layers/cache_identity.py b/superset/semantic_layers/cache_identity.py
new file mode 100644
index 00000000000..759ac0a7046
--- /dev/null
+++ b/superset/semantic_layers/cache_identity.py
@@ -0,0 +1,318 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Immutable identity values for semantic containment caching."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping as MappingABC
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta
+from typing import Mapping
+
+from superset_core.semantic_layers.types import (
+ AdhocExpression,
+ Dimension,
+ Filter,
+ GroupLimit,
+ Metric,
+ OrderTuple,
+ SemanticQuery,
+)
+
+from superset.utils import json
+
+# Prefixes every bucket and value key. Bump it whenever the pickled shape of
+# ``cache_types.CachedEntry`` or the canonical query material changes.
+IDENTITY_FORMAT_VERSION: str = "v4"
+_SENSITIVE_KEY_PARTS: tuple[str, ...] = (
+ "credential",
+ "password",
+ "secret",
+ "token",
+)
+
+
+class SensitiveIdentityMaterialError(ValueError):
+ """Raised when raw secret-like material is offered for cache identity."""
+
+
+def _find_sensitive_paths(value: object, prefix: str = "") -> list[str]:
+ sensitive_paths: list[str] = []
+ if isinstance(value, MappingABC):
+ for key, nested_value in value.items():
+ key_text: str = str(key)
+ path: str = f"{prefix}.{key_text}" if prefix else key_text
+ if any(part in key_text.casefold() for part in _SENSITIVE_KEY_PARTS):
+ sensitive_paths.append(path)
+ sensitive_paths.extend(_find_sensitive_paths(nested_value, path))
+ elif isinstance(value, (list, tuple)):
+ for index, nested_value in enumerate(value):
+ path = f"{prefix}[{index}]"
+ sensitive_paths.extend(_find_sensitive_paths(nested_value, path))
+ return sensitive_paths
+
+
+@dataclass(frozen=True)
+class SemanticViewIdentity:
+ """Stable identity of a semantic view."""
+
+ value: str
+
+
+@dataclass(frozen=True)
+class SemanticDefinitionIdentity:
+ """Versioned digest of result-affecting semantic-view definition fields."""
+
+ digest: str
+
+
+@dataclass(frozen=True)
+class SemanticCacheProviderIdentity:
+ """Versioned digest of result-affecting provider configuration."""
+
+ digest: str
+
+
+@dataclass(frozen=True)
+class SemanticCacheScopeIdentity:
+ """Digest separating global or execution-context cache scopes."""
+
+ digest: str
+
+
+class SemanticCacheIdentityFactory:
+ """Create deterministic, versioned identities from secret-safe material."""
+
+ @classmethod
+ def definition(
+ cls,
+ material: Mapping[str, object],
+ ) -> SemanticDefinitionIdentity:
+ """Create a semantic-definition identity."""
+ return SemanticDefinitionIdentity(cls._digest(material))
+
+ @classmethod
+ def provider(
+ cls,
+ material: Mapping[str, object],
+ ) -> SemanticCacheProviderIdentity:
+ """Create a provider-configuration identity."""
+ return SemanticCacheProviderIdentity(cls._digest(material))
+
+ @classmethod
+ def scope(
+ cls,
+ material: Mapping[str, object],
+ ) -> SemanticCacheScopeIdentity:
+ """Create an execution-scope identity."""
+ return SemanticCacheScopeIdentity(cls._digest(material))
+
+ @classmethod
+ def query(cls, query: SemanticQuery) -> str:
+ """Create a logical identity from every result-affecting query field."""
+ digest: str = cls._digest(_canonical_query(query))
+ return f"semantic-cache:query:{digest}"
+
+ @classmethod
+ def bucket(
+ cls,
+ view: SemanticViewIdentity,
+ definition: SemanticDefinitionIdentity,
+ provider: SemanticCacheProviderIdentity,
+ scope: SemanticCacheScopeIdentity,
+ ) -> str:
+ """Create a descriptor-bucket identity from all host boundaries."""
+ digest: str = cls._digest(
+ {
+ "view": view.value,
+ "definition": definition.digest,
+ "provider": provider.digest,
+ "scope": scope.digest,
+ }
+ )
+ return f"semantic-cache:bucket:{digest}"
+
+ @classmethod
+ def value(cls, bucket: str, query: SemanticQuery) -> str:
+ """Create a result-value identity within a host boundary bucket."""
+ digest: str = cls._digest({"bucket": bucket, "query": cls.query(query)})
+ return f"semantic-cache:value:{digest}"
+
+ @staticmethod
+ def order(order: list[OrderTuple] | None) -> str:
+ """Serialize ordering for descriptor compatibility checks."""
+ if not order:
+ return ""
+ return _canonical_json(
+ [
+ {
+ "element": _canonical_orderable(element),
+ "direction": direction.value,
+ }
+ for element, direction in order
+ ]
+ )
+
+ @staticmethod
+ def group_limit(group_limit: GroupLimit | None) -> str:
+ """Serialize group-limit behavior for descriptor compatibility checks."""
+ if group_limit is None:
+ return ""
+ return _canonical_json(_canonical_group_limit(group_limit))
+
+ @staticmethod
+ def _digest(material: Mapping[str, object]) -> str:
+ sensitive_paths: list[str] = _find_sensitive_paths(material)
+ if sensitive_paths:
+ joined_keys: str = ", ".join(sorted(sensitive_paths))
+ raise SensitiveIdentityMaterialError(
+ f"Secret-like identity fields are not allowed: {joined_keys}"
+ )
+
+ canonical: str = json.dumps(
+ material,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
+ payload: bytes = f"{IDENTITY_FORMAT_VERSION}:{canonical}".encode()
+ return f"{IDENTITY_FORMAT_VERSION}:{hashlib.sha256(payload).hexdigest()}"
+
+
+def _canonical_query(query: SemanticQuery) -> Mapping[str, object]:
+ return {
+ "metrics": [_canonical_metric(metric) for metric in query.metrics],
+ "dimensions": [
+ _canonical_dimension(dimension) for dimension in query.dimensions
+ ],
+ "filters": _canonical_filters(query.filters),
+ "order": [
+ {
+ "element": _canonical_orderable(element),
+ "direction": direction.value,
+ }
+ for element, direction in query.order or []
+ ],
+ "limit": query.limit,
+ "offset": query.offset or 0,
+ "group_limit": _canonical_group_limit(query.group_limit),
+ }
+
+
+def _canonical_metric(metric: Metric) -> Mapping[str, object]:
+ return {
+ "id": metric.id,
+ "name": metric.name,
+ "type": str(metric.type),
+ "definition": metric.definition,
+ "aggregation": metric.aggregation.value if metric.aggregation else None,
+ }
+
+
+def _canonical_dimension(dimension: Dimension) -> Mapping[str, object]:
+ return {
+ "id": dimension.id,
+ "name": dimension.name,
+ "type": str(dimension.type),
+ "definition": dimension.definition,
+ "grain": (
+ {
+ "name": dimension.grain.name,
+ "representation": dimension.grain.representation,
+ }
+ if dimension.grain
+ else None
+ ),
+ }
+
+
+def semantic_dimension_key(dimension: Dimension) -> str:
+ """Return a dimension identity that distinguishes requested grains."""
+ if dimension.grain is None:
+ return dimension.id
+ return f"{dimension.id}@{dimension.grain.representation}"
+
+
+def _canonical_filter(filter_: Filter) -> Mapping[str, object]:
+ return {
+ "type": filter_.type.value,
+ "column": filter_.column.id if filter_.column else None,
+ "operator": filter_.operator.value,
+ "value": _canonical_value(filter_.value),
+ }
+
+
+def _canonical_filters(filters: set[Filter] | None) -> list[Mapping[str, object]]:
+ canonical: list[Mapping[str, object]] = [
+ _canonical_filter(filter_) for filter_ in filters or set()
+ ]
+ return sorted(canonical, key=_canonical_json)
+
+
+def _canonical_value(value: object) -> Mapping[str, object]:
+ if isinstance(value, frozenset):
+ items: list[Mapping[str, object]] = [_canonical_value(item) for item in value]
+ return {"type": "frozenset", "value": sorted(items, key=_canonical_json)}
+ if isinstance(value, tuple):
+ return {"type": "tuple", "value": [_canonical_value(item) for item in value]}
+ if isinstance(value, datetime):
+ return {"type": "datetime", "value": value.isoformat()}
+ if isinstance(value, date):
+ return {"type": "date", "value": value.isoformat()}
+ if isinstance(value, time):
+ return {"type": "time", "value": value.isoformat()}
+ if isinstance(value, timedelta):
+ return {"type": "timedelta", "value": value.total_seconds()}
+ return {"type": type(value).__name__, "value": value}
+
+
+def _canonical_orderable(
+ element: Metric | Dimension | AdhocExpression,
+) -> Mapping[str, object]:
+ if isinstance(element, Metric):
+ return {"kind": "metric", **_canonical_metric(element)}
+ if isinstance(element, Dimension):
+ return {"kind": "dimension", **_canonical_dimension(element)}
+ return {"kind": "adhoc", "id": element.id, "definition": element.definition}
+
+
+def _canonical_group_limit(group_limit: GroupLimit | None) -> object:
+ if group_limit is None:
+ return None
+ return {
+ "dimensions": [
+ _canonical_dimension(dimension) for dimension in group_limit.dimensions
+ ],
+ "top": group_limit.top,
+ "metric": (
+ _canonical_metric(group_limit.metric) if group_limit.metric else None
+ ),
+ "direction": group_limit.direction.value,
+ "group_others": group_limit.group_others,
+ "filters": _canonical_filters(group_limit.filters),
+ }
+
+
+def _canonical_json(value: object) -> str:
+ return json.dumps(
+ value,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
diff --git a/superset/semantic_layers/cache_policy.py b/superset/semantic_layers/cache_policy.py
new file mode 100644
index 00000000000..43a410dc060
--- /dev/null
+++ b/superset/semantic_layers/cache_policy.py
@@ -0,0 +1,394 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Pure policy types for semantic containment caching."""
+
+from __future__ import annotations
+
+import re
+
+from superset_core.semantic_layers.types import (
+ Dimension,
+ Filter,
+ Metric,
+ Operator,
+ PredicateType,
+ SemanticQuery,
+)
+
+from superset.semantic_layers.cache_identity import SemanticCacheIdentityFactory
+from superset.semantic_layers.cache_types import (
+ CachedEntry as CachedEntry,
+ ContainmentCapabilities as ContainmentCapabilities,
+ PatternSemantics as PatternSemantics,
+ ReuseDecision as ReuseDecision,
+ ReuseMode as ReuseMode,
+ ROLLUP_COMPATIBLE_AGGREGATIONS,
+ SAFE_CONTAINMENT_CAPABILITIES as SAFE_CONTAINMENT_CAPABILITIES,
+)
+
+
+def _comparable(left: object, right: object) -> bool:
+ return type(left) is type(right) or (
+ isinstance(left, (int, float))
+ and not isinstance(left, bool)
+ and isinstance(right, (int, float))
+ and not isinstance(right, bool)
+ )
+
+
+def _range_contains(operator: Operator, bound: object, value: object) -> bool:
+ if not _comparable(bound, value):
+ return False
+ if operator is Operator.GREATER_THAN:
+ return bool(value > bound) # type: ignore[operator]
+ if operator is Operator.GREATER_THAN_OR_EQUAL:
+ return bool(value >= bound) # type: ignore[operator]
+ if operator is Operator.LESS_THAN:
+ return bool(value < bound) # type: ignore[operator]
+ if operator is Operator.LESS_THAN_OR_EQUAL:
+ return bool(value <= bound) # type: ignore[operator]
+ return False
+
+
+def compile_like_pattern(pattern: str, escape: str) -> re.Pattern[str]:
+ """Compile a SQL LIKE pattern using the provider's escape character."""
+ parts: list[str] = []
+ index: int = 0
+ while index < len(pattern):
+ char: str = pattern[index]
+ if char == escape and index + 1 < len(pattern):
+ index += 1
+ parts.append(re.escape(pattern[index]))
+ elif char == "%":
+ parts.append(".*")
+ while index + 1 < len(pattern) and pattern[index + 1] == "%":
+ index += 1
+ elif char == "_":
+ parts.append(".")
+ else:
+ parts.append(re.escape(char))
+ index += 1
+ return re.compile("^" + "".join(parts) + "$", re.DOTALL)
+
+
+_RANGE_OPERATORS: frozenset[Operator] = frozenset(
+ {
+ Operator.GREATER_THAN,
+ Operator.GREATER_THAN_OR_EQUAL,
+ Operator.LESS_THAN,
+ Operator.LESS_THAN_OR_EQUAL,
+ }
+)
+_LOWER_BOUND_OPERATORS: frozenset[Operator] = frozenset(
+ {Operator.GREATER_THAN, Operator.GREATER_THAN_OR_EQUAL}
+)
+
+
+def _implies_membership(new_filter: Filter, cached_values: object) -> bool:
+ if not isinstance(cached_values, (set, frozenset, tuple)):
+ return False
+ if new_filter.operator is Operator.EQUALS:
+ return any(
+ _comparable(new_filter.value, cached_value)
+ and new_filter.value == cached_value
+ for cached_value in cached_values
+ )
+ if new_filter.operator is Operator.IN and isinstance(
+ new_filter.value, (set, frozenset, tuple)
+ ):
+ return all(
+ any(
+ _comparable(new_value, cached_value) and new_value == cached_value
+ for cached_value in cached_values
+ )
+ for new_value in new_filter.value
+ )
+ return False
+
+
+def _implies_range(new_filter: Filter, cached_filter: Filter) -> bool:
+ if new_filter.operator is Operator.EQUALS:
+ return _range_contains(
+ cached_filter.operator, cached_filter.value, new_filter.value
+ )
+ if new_filter.operator is Operator.IN and isinstance(
+ new_filter.value, (set, frozenset, tuple)
+ ):
+ return all(
+ _range_contains(cached_filter.operator, cached_filter.value, value)
+ for value in new_filter.value
+ )
+ if new_filter.operator not in _RANGE_OPERATORS or not _range_contains(
+ cached_filter.operator, cached_filter.value, new_filter.value
+ ):
+ return False
+ cached_is_lower: bool = cached_filter.operator in _LOWER_BOUND_OPERATORS
+ new_is_lower: bool = new_filter.operator in _LOWER_BOUND_OPERATORS
+ return cached_is_lower is new_is_lower
+
+
+def _implies_like(
+ new_filter: Filter,
+ cached_filter: Filter,
+ pattern_semantics: PatternSemantics | None,
+) -> bool:
+ return (
+ new_filter.operator is Operator.EQUALS
+ and isinstance(new_filter.value, str)
+ and isinstance(cached_filter.value, str)
+ and pattern_semantics is not None
+ and compile_like_pattern(
+ cached_filter.value, pattern_semantics.escape
+ ).fullmatch(new_filter.value)
+ is not None
+ )
+
+
+def implies(
+ new_filter: Filter,
+ cached_filter: Filter,
+ *,
+ pattern_semantics: PatternSemantics | None = None,
+) -> bool:
+ """Return whether the new predicate proves the cached predicate."""
+ if (
+ new_filter.type is not cached_filter.type
+ or new_filter.column != cached_filter.column
+ ):
+ return False
+ new_op: Operator = new_filter.operator
+ cached_op: Operator = cached_filter.operator
+ new_value: object = new_filter.value
+ cached_value: object = cached_filter.value
+ if (
+ new_op is cached_op
+ and _comparable(new_value, cached_value)
+ and new_value == cached_value
+ ):
+ return True
+ if cached_op is Operator.IS_NULL:
+ return False
+ if cached_op is Operator.IS_NOT_NULL:
+ return new_op is not Operator.IS_NULL
+ if cached_op is Operator.EQUALS:
+ return (
+ new_op is Operator.EQUALS
+ and _comparable(new_value, cached_value)
+ and new_value == cached_value
+ )
+ if cached_op is Operator.IN:
+ return _implies_membership(new_filter, cached_value)
+ if cached_op in _RANGE_OPERATORS:
+ return _implies_range(new_filter, cached_filter)
+ if cached_op is Operator.LIKE:
+ return _implies_like(new_filter, cached_filter, pattern_semantics)
+ return False
+
+
+def _supports_filter(
+ filter_: Filter,
+ capabilities: ContainmentCapabilities,
+) -> bool:
+ operator: Operator = filter_.operator
+ if operator in {
+ Operator.EQUALS,
+ Operator.NOT_EQUALS,
+ *_RANGE_OPERATORS,
+ }:
+ return capabilities.comparisons
+ if operator in {Operator.IN, Operator.NOT_IN}:
+ return capabilities.membership
+ if operator in {Operator.IS_NULL, Operator.IS_NOT_NULL}:
+ return capabilities.nulls
+ if operator in {Operator.LIKE, Operator.NOT_LIKE}:
+ return capabilities.pattern_semantics is not None
+ return False
+
+
+def _strict_filters(filters: frozenset[Filter]) -> frozenset[Filter]:
+ return frozenset(
+ filter_
+ for filter_ in filters
+ if filter_.type is PredicateType.HAVING or filter_.operator is Operator.ADHOC
+ )
+
+
+def _same_filter(left: Filter, right: Filter) -> bool:
+ """Require compatible literal types even when dataclass equality matches."""
+ return left == right and _comparable(left.value, right.value)
+
+
+def _filter_decision(
+ query: SemanticQuery,
+ entry: CachedEntry,
+ capabilities: ContainmentCapabilities,
+) -> frozenset[Filter] | None:
+ requested: frozenset[Filter] = frozenset(query.filters or set())
+ cached: frozenset[Filter] = entry.filters
+ requested_strict: frozenset[Filter] = _strict_filters(requested)
+ cached_strict: frozenset[Filter] = _strict_filters(cached)
+ if requested_strict != cached_strict:
+ return None
+
+ requested_where: frozenset[Filter] = requested - requested_strict
+ cached_where: frozenset[Filter] = cached - cached_strict
+ for cached_filter in cached_where:
+ if any(
+ _same_filter(cached_filter, requested_filter)
+ for requested_filter in requested_where
+ ):
+ continue
+ if not any(
+ _supports_filter(requested_filter, capabilities)
+ and _supports_filter(cached_filter, capabilities)
+ and implies(
+ requested_filter,
+ cached_filter,
+ pattern_semantics=capabilities.pattern_semantics,
+ )
+ for requested_filter in requested_where
+ ):
+ return None
+
+ leftovers: frozenset[Filter] = frozenset(
+ filter_
+ for filter_ in requested_where
+ if not any(
+ _same_filter(filter_, cached_filter) for cached_filter in cached_where
+ )
+ )
+ # Cached time axes contain bucket starts, not the raw values used by WHERE.
+ grained_columns: set[str] = {
+ dimension.name for dimension in entry.dimensions if dimension.grain is not None
+ }
+ if any(
+ not isinstance(leftover.column, Dimension)
+ or leftover.column.grain is not None
+ or leftover.column.name in grained_columns
+ or leftover.column not in entry.dimensions
+ or not _supports_filter(leftover, capabilities)
+ for leftover in leftovers
+ ):
+ return None
+ return leftovers
+
+
+def _reuse_mode(query: SemanticQuery, entry: CachedEntry) -> ReuseMode | None:
+ requested_dimensions: frozenset[Dimension] = frozenset(query.dimensions)
+ requested_metrics: frozenset[Metric] = frozenset(query.metrics)
+ if not requested_metrics.issubset(entry.metrics):
+ return None
+ if requested_dimensions == entry.dimensions:
+ return (
+ ReuseMode.EXACT if requested_metrics == entry.metrics else ReuseMode.PROJECT
+ )
+ if not requested_dimensions < entry.dimensions:
+ return None
+ return (
+ ReuseMode.ROLLUP
+ if all(
+ metric.aggregation in ROLLUP_COMPATIBLE_AGGREGATIONS
+ for metric in query.metrics
+ )
+ else None
+ )
+
+
+def select_reuse(
+ query: SemanticQuery,
+ entry: CachedEntry,
+ capabilities: ContainmentCapabilities,
+) -> ReuseDecision | None:
+ """Prove whether and how a cached descriptor can satisfy a query."""
+ query_order_key: str = SemanticCacheIdentityFactory.order(query.order)
+ query_offset: int = query.offset or 0
+ pagination_compatible: bool = (
+ query_offset == 0
+ and entry.offset == 0
+ and query_order_key == entry.order_key
+ and (
+ entry.limit is None
+ or (query.limit is not None and query.limit <= entry.limit)
+ )
+ )
+ group_limit_compatible: bool = (
+ SemanticCacheIdentityFactory.group_limit(query.group_limit)
+ == entry.group_limit_key
+ )
+ if not pagination_compatible or not group_limit_compatible:
+ return None
+ leftovers: frozenset[Filter] | None = _filter_decision(query, entry, capabilities)
+ mode: ReuseMode | None = _reuse_mode(query, entry)
+ if leftovers is None or mode is None:
+ return None
+ if mode is ReuseMode.ROLLUP and query.order:
+ return None
+ if mode is ReuseMode.ROLLUP and _strict_filters(frozenset(query.filters or set())):
+ return None
+ if query.group_limit is not None and leftovers:
+ return None
+ if entry.limit is not None and (leftovers or mode is ReuseMode.ROLLUP):
+ return None
+ return ReuseDecision(mode=mode, leftover_filters=leftovers)
+
+
+def eligible_reuse_mode(
+ query: SemanticQuery,
+ entry: CachedEntry,
+ capabilities: ContainmentCapabilities = SAFE_CONTAINMENT_CAPABILITIES,
+) -> ReuseMode | None:
+ """Return only the transformation mode for an eligible candidate."""
+ decision: ReuseDecision | None = select_reuse(query, entry, capabilities)
+ return decision.mode if decision else None
+
+
+def rank_reuse_decisions(
+ query: SemanticQuery,
+ entries: list[CachedEntry],
+ capabilities: ContainmentCapabilities = SAFE_CONTAINMENT_CAPABILITIES,
+) -> list[tuple[ReuseDecision, CachedEntry]]:
+ """Rank entries with the reuse proof already calculated for each one."""
+ priorities: dict[ReuseMode, int] = {
+ ReuseMode.EXACT: 0,
+ ReuseMode.PROJECT: 1,
+ ReuseMode.ROLLUP: 2,
+ }
+ candidates: list[tuple[ReuseDecision, CachedEntry]] = []
+ for entry in entries:
+ decision: ReuseDecision | None = select_reuse(query, entry, capabilities)
+ if decision is not None:
+ candidates.append((decision, entry))
+ return sorted(
+ candidates,
+ key=lambda candidate: (
+ priorities[candidate[0].mode],
+ -candidate[1].timestamp,
+ ),
+ )
+
+
+def rank_eligible_entries(
+ query: SemanticQuery,
+ entries: list[CachedEntry],
+ capabilities: ContainmentCapabilities = SAFE_CONTAINMENT_CAPABILITIES,
+) -> list[tuple[ReuseMode, CachedEntry]]:
+ """Rank eligible candidates by transformation cost and freshness."""
+ return [
+ (decision.mode, entry)
+ for decision, entry in rank_reuse_decisions(query, entries, capabilities)
+ ]
diff --git a/superset/semantic_layers/cache_repository.py b/superset/semantic_layers/cache_repository.py
new file mode 100644
index 00000000000..473b43bc3e0
--- /dev/null
+++ b/superset/semantic_layers/cache_repository.py
@@ -0,0 +1,450 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Storage records for semantic containment caching."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from contextvars import ContextVar
+from dataclasses import dataclass, fields
+from math import ceil
+from time import time
+from typing import cast, Protocol, TYPE_CHECKING, TypeGuard
+from uuid import uuid4
+
+if TYPE_CHECKING:
+ from superset.coordination.cache_backend import RedisCommandsMixin
+
+from superset_core.semantic_layers.types import SemanticQuery, SemanticResult
+
+from superset.semantic_layers.cache_identity import (
+ SemanticCacheIdentityFactory,
+ SemanticCacheProviderIdentity,
+ SemanticCacheScopeIdentity,
+ SemanticDefinitionIdentity,
+ SemanticViewIdentity,
+)
+from superset.semantic_layers.cache_policy import rank_reuse_decisions
+from superset.semantic_layers.cache_types import (
+ CachedEntry as CachedEntry,
+ CachedResultCandidate as CachedResultCandidate,
+ CachedValue,
+ ContainmentCapabilities,
+ ReuseDecision,
+ SemanticCacheLookupResult as SemanticCacheLookupResult,
+)
+
+MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET: int = 128
+
+
+class SemanticCacheRepositoryError(RuntimeError):
+ """Base error for expected cache-adapter failures."""
+
+
+class SemanticCacheLookupError(SemanticCacheRepositoryError):
+ """Raised when an expected backend lookup operation fails."""
+
+
+class SemanticCacheStoreError(SemanticCacheRepositoryError):
+ """Raised when an expected backend store operation fails."""
+
+
+class SemanticCacheBackendError(RuntimeError):
+ """Expected operational failure raised by an expiring backend adapter."""
+
+
+class SemanticCacheCoordinationError(RuntimeError):
+ """Expected operational failure raised by a mutation coordinator."""
+
+
+@dataclass(frozen=True)
+class SemanticCacheWriteFence:
+ """Mutation-local lease identity; never shared between concurrent writers."""
+
+ key: str
+ owner_token: str
+ check_owner: Callable[[], bool]
+ redis_backend: RedisCommandsMixin | None = None
+
+
+semantic_cache_write_fence: ContextVar[SemanticCacheWriteFence | None] = ContextVar(
+ "semantic_cache_write_fence", default=None
+)
+
+
+@dataclass(frozen=True)
+class ViewMeta:
+ """Identity and expiry inputs for one semantic-view cache bucket."""
+
+ view_identity: SemanticViewIdentity
+ definition_identity: SemanticDefinitionIdentity
+ provider_identity: SemanticCacheProviderIdentity
+ scope_identity: SemanticCacheScopeIdentity
+ timeout: int | None
+
+
+class SemanticCacheBackend(Protocol):
+ """Expiring value/descriptor operations required by the repository."""
+
+ def get(self, key: str) -> object | None: ... # pragma: no cover
+
+ def has(self, key: str) -> bool: ... # pragma: no cover
+
+ def set(
+ self, key: str, value: object, timeout: int | None = None
+ ) -> bool: ... # pragma: no cover
+
+ def delete(self, key: str) -> bool: ... # pragma: no cover
+
+
+class SemanticCacheMutationCoordinator(Protocol):
+ """Boundary for ownership-safe descriptor mutation."""
+
+ def mutate(
+ self, key: str, operation: Callable[[], None]
+ ) -> bool: ... # pragma: no cover
+
+
+class SemanticCacheRepository:
+ """Store expiring results and bounded descriptors behind injected ports."""
+
+ def __init__(
+ self,
+ backend: SemanticCacheBackend,
+ coordinator: SemanticCacheMutationCoordinator,
+ *,
+ clock: Callable[[], float] = time,
+ ) -> None:
+ self._backend: SemanticCacheBackend = backend
+ self._coordinator: SemanticCacheMutationCoordinator = coordinator
+ self._clock: Callable[[], float] = clock
+
+ def _get(
+ self,
+ key: str,
+ error_type: type[SemanticCacheRepositoryError],
+ ) -> object | None:
+ try:
+ return self._backend.get(key)
+ except SemanticCacheBackendError as ex:
+ raise error_type("Semantic cache backend get failed") from ex
+
+ def _has(
+ self,
+ key: str,
+ error_type: type[SemanticCacheRepositoryError],
+ ) -> bool:
+ try:
+ return self._backend.has(key)
+ except SemanticCacheBackendError as ex:
+ raise error_type("Semantic cache backend has failed") from ex
+
+ def _set(
+ self,
+ key: str,
+ value: object,
+ timeout: int | None,
+ error_type: type[SemanticCacheRepositoryError],
+ ) -> None:
+ try:
+ fence: SemanticCacheWriteFence | None = semantic_cache_write_fence.get()
+ persisted: bool | None = None
+ if fence is not None:
+ fenced_set: (
+ Callable[
+ [SemanticCacheWriteFence, str, object, int | None], bool | None
+ ]
+ | None
+ ) = getattr(self._backend, "set_if_owner", None)
+ if fenced_set is not None:
+ persisted = fenced_set(fence, key, value, timeout)
+ if persisted is None and not fence.check_owner():
+ raise SemanticCacheCoordinationError(
+ "Semantic cache lease ownership was lost"
+ )
+ if persisted is None:
+ # Non-Redis or separate-client stores cannot atomically check
+ # the lease and SET. Recheck immediately before this best-effort
+ # write. Readers reject mismatched descriptor/value nonces if a
+ # lease takeover allows a stale SET to land.
+ persisted = self._backend.set(key, value, timeout=timeout)
+ except SemanticCacheBackendError as ex:
+ raise error_type("Semantic cache backend set failed") from ex
+ if not persisted:
+ raise error_type("Semantic cache backend rejected set")
+
+ def _mutate(
+ self,
+ key: str,
+ operation: Callable[[], None],
+ error_type: type[SemanticCacheRepositoryError],
+ ) -> bool:
+ try:
+ return self._coordinator.mutate(key, operation)
+ except SemanticCacheCoordinationError as ex:
+ raise error_type("Semantic cache descriptor mutation failed") from ex
+
+ def _delete(
+ self,
+ key: str,
+ error_type: type[SemanticCacheRepositoryError],
+ ) -> None:
+ try:
+ self._backend.delete(key)
+ except SemanticCacheBackendError as ex:
+ raise error_type("Semantic cache backend delete failed") from ex
+
+ @staticmethod
+ def _bucket_key(meta: ViewMeta) -> str:
+ return SemanticCacheIdentityFactory.bucket(
+ meta.view_identity,
+ meta.definition_identity,
+ meta.provider_identity,
+ meta.scope_identity,
+ )
+
+ @staticmethod
+ def _entries(value: object | None) -> list[CachedEntry]:
+ # A pickled dataclass restores only the attributes it was written
+ # with, so an entry from an older shape can pass ``isinstance`` and
+ # still lack a field (a class-level default masks that from
+ # ``hasattr``, hence the instance-dict check). Key rotation
+ # (``IDENTITY_FORMAT_VERSION``) is the primary defence; treating such
+ # a bucket as empty is the backstop, so it costs one miss and a
+ # re-store rather than a lookup failure on every request until the
+ # bucket expires.
+ expected: tuple[str, ...] = tuple(field.name for field in fields(CachedEntry))
+ if not isinstance(value, list) or not all(
+ isinstance(entry, CachedEntry)
+ and all(name in vars(entry) for name in expected)
+ for entry in value
+ ):
+ return []
+ return cast(list[CachedEntry], value)
+
+ def store(
+ self,
+ meta: ViewMeta,
+ query: SemanticQuery,
+ result: SemanticResult,
+ *,
+ replace: bool = True,
+ ) -> bool:
+ """Store a TTL-bounded value and register its bounded descriptor.
+
+ With ``replace`` false, a value that another request already stored
+ and registered under the same identity is left alone: the identical
+ provider result is already discoverable, and skipping saves this
+ request the lease wait and the payload write. A herd of identical
+ misses then contends for the lease once, not once per request.
+ """
+ bucket_key: str = self._bucket_key(meta)
+ value_key: str = SemanticCacheIdentityFactory.value(bucket_key, query)
+ now: float = self._clock()
+ if not replace and self._is_registered(bucket_key, value_key, meta, now):
+ return True
+ descriptor: CachedEntry = CachedEntry(
+ filters=frozenset(query.filters or set()),
+ dimensions=frozenset(query.dimensions),
+ metrics=frozenset(query.metrics),
+ limit=query.limit,
+ offset=query.offset or 0,
+ order_key=SemanticCacheIdentityFactory.order(query.order),
+ group_limit_key=SemanticCacheIdentityFactory.group_limit(query.group_limit),
+ value_key=value_key,
+ timestamp=now,
+ timeout=meta.timeout,
+ write_nonce=uuid4().hex,
+ )
+
+ def register() -> None:
+ entries: list[CachedEntry] = self._entries(
+ self._get(bucket_key, SemanticCacheStoreError)
+ )
+ retained: list[CachedEntry] = [
+ entry for entry in entries if entry.value_key != value_key
+ ]
+ retained.append(descriptor)
+ bounded: list[CachedEntry] = sorted(
+ retained,
+ key=lambda entry: entry.timestamp,
+ reverse=True,
+ )[:MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET]
+ bounded_value_keys: set[str] = {entry.value_key for entry in bounded}
+ evicted_value_keys: set[str] = {
+ entry.value_key
+ for entry in retained
+ if entry.value_key not in bounded_value_keys
+ }
+ self._set(
+ bucket_key,
+ bounded,
+ self._bucket_timeout(bounded, meta.timeout, now),
+ SemanticCacheStoreError,
+ )
+ self._set(
+ value_key,
+ CachedValue(descriptor.write_nonce, result),
+ meta.timeout,
+ SemanticCacheStoreError,
+ )
+ for evicted_value_key in evicted_value_keys:
+ self._delete(evicted_value_key, SemanticCacheStoreError)
+
+ return self._mutate(bucket_key, register, SemanticCacheStoreError)
+
+ @staticmethod
+ def _serves(entry: CachedEntry, meta: ViewMeta, now: float) -> bool:
+ """Whether a stored value may answer a request with ``meta``'s timeout.
+
+ Requests resolve their own cache timeout, so a value stored by a
+ long-timeout chart can outlive what a short-timeout chart on the same
+ view is allowed to reuse. The value's age is bounded by the requesting
+ timeout (``None`` and ``0`` follow the backend's unbounded conventions),
+ and for the store dedupe the stored TTL must also cover this request,
+ or the longer-lived chart would be pinned to the shorter entry.
+ """
+ if meta.timeout is None or meta.timeout <= 0:
+ return True
+ if now - entry.timestamp > meta.timeout:
+ return False
+ return (
+ entry.timeout is None
+ or entry.timeout <= 0
+ or (entry.timeout >= meta.timeout)
+ )
+
+ def _is_registered(
+ self,
+ bucket_key: str,
+ value_key: str,
+ meta: ViewMeta,
+ now: float,
+ ) -> bool:
+ entries: list[CachedEntry] = self._entries(
+ self._get(bucket_key, SemanticCacheStoreError)
+ )
+ return any(
+ entry.value_key == value_key
+ and self._serves(entry, meta, now)
+ and self._has(value_key, SemanticCacheStoreError)
+ and self._matches(self._get(value_key, SemanticCacheStoreError), entry)
+ for entry in entries
+ )
+
+ @staticmethod
+ def _matches(value: object, entry: CachedEntry) -> TypeGuard[CachedValue]:
+ """Never serve or deduplicate an unfenced or mismatched publication."""
+ return (
+ isinstance(value, CachedValue)
+ and bool(entry.write_nonce)
+ and entry.write_nonce == getattr(value, "write_nonce", None)
+ and isinstance(getattr(value, "result", None), SemanticResult)
+ )
+
+ def lookup(
+ self,
+ meta: ViewMeta,
+ query: SemanticQuery,
+ capabilities: ContainmentCapabilities,
+ ) -> SemanticCacheLookupResult:
+ """Load ranked values and report missing descriptors without mutation."""
+ bucket_key: str = self._bucket_key(meta)
+ now: float = self._clock()
+ entries: list[CachedEntry] = [
+ entry
+ for entry in self._entries(self._get(bucket_key, SemanticCacheLookupError))
+ if self._serves(entry, meta, now)
+ ]
+ ranked: list[tuple[ReuseDecision, CachedEntry]] = rank_reuse_decisions(
+ query, entries, capabilities
+ )
+ candidates: list[CachedResultCandidate] = []
+ missing_value_keys: set[str] = set()
+ fence_rejections: int = 0
+ for decision, entry in ranked:
+ value: object | None = self._get(entry.value_key, SemanticCacheLookupError)
+ if not self._matches(value, entry):
+ missing_value_keys.add(entry.value_key)
+ fence_rejections += int(value is not None)
+ continue
+ candidates.append(CachedResultCandidate(entry, value.result, decision))
+ break
+ return SemanticCacheLookupResult(
+ candidates=tuple(candidates),
+ missing_value_keys=frozenset(missing_value_keys),
+ fence_rejections=fence_rejections,
+ )
+
+ def prune_missing(
+ self,
+ meta: ViewMeta,
+ missing_value_keys: frozenset[str],
+ ) -> None:
+ """Recheck and prune invalid descriptors without deleting payloads."""
+ if not missing_value_keys:
+ return
+ bucket_key: str = self._bucket_key(meta)
+ now: float = self._clock()
+
+ def prune() -> None:
+ entries: list[CachedEntry] = self._entries(
+ self._get(bucket_key, SemanticCacheLookupError)
+ )
+ retained: list[CachedEntry] = [
+ entry
+ for entry in entries
+ if entry.value_key not in missing_value_keys
+ or self._matches(
+ self._get(entry.value_key, SemanticCacheLookupError), entry
+ )
+ ]
+ self._set(
+ bucket_key,
+ retained,
+ self._bucket_timeout(retained, meta.timeout, now),
+ SemanticCacheLookupError,
+ )
+
+ self._mutate(bucket_key, prune, SemanticCacheLookupError)
+
+ @staticmethod
+ def _bucket_timeout(
+ entries: list[CachedEntry],
+ fallback: int | None,
+ now: float,
+ ) -> int | None:
+ """Expire a bucket no sooner than the longest-lived value it indexes.
+
+ Requests resolve their own cache timeout (custom, chart, then dataset),
+ so one view's bucket indexes values stored with different TTLs. Writing
+ the bucket with the latest request's timeout would let a short-lived
+ store expire the index out from under longer-lived values, orphaning
+ them until their own TTL. ``0`` follows the backend's never-expire
+ convention. An unknown backend-default TTL (``None``) also requires
+ an unbounded index so an explicit shorter TTL cannot orphan its value.
+ """
+ timeouts: list[int | None] = [entry.timeout for entry in entries]
+ if not timeouts:
+ return fallback
+ if any(timeout is None or timeout <= 0 for timeout in timeouts):
+ return 0
+ remaining: list[float] = [
+ entry.timestamp + (entry.timeout or 0) - now for entry in entries
+ ]
+ return max(1, ceil(max(remaining)))
diff --git a/superset/semantic_layers/cache_transform.py b/superset/semantic_layers/cache_transform.py
new file mode 100644
index 00000000000..42829c3dfe7
--- /dev/null
+++ b/superset/semantic_layers/cache_transform.py
@@ -0,0 +1,268 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""In-memory transformations for proven-compatible semantic cache results."""
+
+from collections.abc import Callable
+from decimal import Context, localcontext
+
+import pandas as pd
+import pyarrow as pa
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Filter,
+ Operator,
+ SemanticQuery,
+ SemanticResult,
+)
+
+from superset.semantic_layers.cache_policy import compile_like_pattern
+from superset.semantic_layers.cache_types import (
+ ContainmentCapabilities,
+ PatternSemantics,
+ ReuseDecision,
+ ReuseMode,
+ ROLLUP_COMPATIBLE_AGGREGATIONS,
+)
+
+
+def _sql_sum(series: pd.Series) -> object:
+ """Sum exact numeric values without fixed-width overflow or decimal rounding."""
+ if series.count() == 0:
+ return None
+ if isinstance(series.dtype, pd.ArrowDtype) and (
+ pa.types.is_integer(series.dtype.pyarrow_dtype)
+ or pa.types.is_decimal(series.dtype.pyarrow_dtype)
+ ):
+ context: Context
+ with localcontext() as context:
+ # Decimal256 inputs have at most 76 digits; summing N values can
+ # require ceil(log10(N)) additional digits before the safe cast.
+ context.prec = 76 + len(str(len(series))) + 1
+ return sum(series.dropna().tolist())
+ return series.sum(min_count=1)
+
+
+def _count_sum(series: pd.Series) -> object:
+ """Combine provider counts while preserving the empty-count value."""
+ return 0 if series.count() == 0 else _sql_sum(series)
+
+
+_ROLLUP_AGGREGATIONS: dict[AggregationType, str | Callable[[pd.Series], object]] = {
+ AggregationType.SUM: _sql_sum,
+ AggregationType.COUNT: _count_sum,
+ AggregationType.MIN: "min",
+ AggregationType.MAX: "max",
+}
+assert frozenset(_ROLLUP_AGGREGATIONS) == ROLLUP_COMPATIBLE_AGGREGATIONS
+
+
+class SemanticCacheTransformationError(RuntimeError):
+ """Cached data cannot safely satisfy an otherwise eligible query."""
+
+
+def _comparison_mask(
+ series: pd.Series, operator: Operator, value: object
+) -> pd.Series | None:
+ if value is None:
+ return pd.Series(False, index=series.index, dtype=bool)
+ not_null: pd.Series = series.notna()
+ comparisons: dict[Operator, Callable[[object], pd.Series]] = {
+ Operator.EQUALS: series.eq,
+ Operator.NOT_EQUALS: series.ne,
+ Operator.GREATER_THAN: series.gt,
+ Operator.GREATER_THAN_OR_EQUAL: series.ge,
+ Operator.LESS_THAN: series.lt,
+ Operator.LESS_THAN_OR_EQUAL: series.le,
+ }
+ comparison: Callable[[object], pd.Series] | None = comparisons.get(operator)
+ return not_null & comparison(value) if comparison else None
+
+
+def _membership_mask(
+ series: pd.Series,
+ operator: Operator,
+ values: set[object] | frozenset[object] | tuple[object, ...],
+) -> pd.Series:
+ not_null: pd.Series = series.notna()
+ non_null_values: list[object] = [item for item in values if item is not None]
+ membership: pd.Series = not_null & series.isin(non_null_values)
+ if operator is Operator.IN:
+ return membership
+ if None in values:
+ return pd.Series(False, index=series.index, dtype=bool)
+ return not_null & ~membership
+
+
+def _pattern_mask(
+ series: pd.Series,
+ operator: Operator,
+ pattern: str,
+ semantics: PatternSemantics,
+) -> pd.Series:
+ not_null: pd.Series = series.notna()
+ matches: pd.Series = not_null & series.astype("string").str.fullmatch(
+ compile_like_pattern(pattern, semantics.escape), na=False
+ )
+ return matches if operator is Operator.LIKE else not_null & ~matches
+
+
+def mask_for(
+ series: pd.Series,
+ operator: Operator,
+ value: object,
+ *,
+ pattern_semantics: PatternSemantics | None = None,
+) -> pd.Series:
+ """Return a SQL-WHERE-compatible boolean mask for a pandas series."""
+ if operator is Operator.IS_NULL:
+ return series.isna()
+ if operator is Operator.IS_NOT_NULL:
+ return series.notna()
+ comparison: pd.Series | None = _comparison_mask(series, operator, value)
+ if comparison is not None:
+ return comparison
+ if operator in {Operator.IN, Operator.NOT_IN} and isinstance(
+ value, (set, frozenset, tuple)
+ ):
+ return _membership_mask(series, operator, value)
+ if (
+ operator in {Operator.LIKE, Operator.NOT_LIKE}
+ and isinstance(value, str)
+ and pattern_semantics is not None
+ ):
+ return _pattern_mask(series, operator, value, pattern_semantics)
+ raise ValueError(f"Unsupported cached filter operation: {operator.value}")
+
+
+def _apply_leftovers(
+ frame: pd.DataFrame,
+ leftovers: frozenset[Filter],
+ capabilities: ContainmentCapabilities,
+) -> pd.DataFrame:
+ transformed: pd.DataFrame = frame
+ for filter_ in sorted(
+ leftovers,
+ key=lambda item: (
+ item.column.id if item.column else "",
+ item.operator.value,
+ ),
+ ):
+ if filter_.column is None:
+ raise ValueError("Cached post-processing requires a filter column")
+ column_name: str = filter_.column.name
+ transformed = transformed.loc[
+ mask_for(
+ transformed[column_name],
+ filter_.operator,
+ filter_.value,
+ pattern_semantics=capabilities.pattern_semantics,
+ )
+ ]
+ return transformed
+
+
+def _rollup(frame: pd.DataFrame, query: SemanticQuery) -> pd.DataFrame:
+ dimension_names: list[str] = [dimension.name for dimension in query.dimensions]
+ aggregations: dict[str, str | Callable[[pd.Series], object]] = {}
+ for metric in query.metrics:
+ if metric.aggregation not in _ROLLUP_AGGREGATIONS:
+ raise ValueError(f"Metric {metric.id} is not safely roll-up compatible")
+ aggregations[metric.name] = _ROLLUP_AGGREGATIONS[metric.aggregation]
+ if dimension_names:
+ return frame.groupby(
+ dimension_names, as_index=False, dropna=False, observed=True
+ ).agg(aggregations)
+ # Transposing a mixed numeric Series can coerce exact integers to floats.
+ # Build each column independently with its original Arrow-backed dtype.
+ return pd.DataFrame(
+ {
+ name: pd.Series([frame[name].agg(aggregation)], dtype=frame[name].dtype)
+ for name, aggregation in aggregations.items()
+ }
+ )
+
+
+def _project(frame: pd.DataFrame, query: SemanticQuery) -> pd.DataFrame:
+ columns: list[str] = [
+ *(dimension.name for dimension in query.dimensions),
+ *(metric.name for metric in query.metrics),
+ ]
+ return frame.loc[:, columns]
+
+
+def _slice(frame: pd.DataFrame, query: SemanticQuery) -> pd.DataFrame:
+ offset: int = query.offset or 0
+ stop: int | None = offset + query.limit if query.limit is not None else None
+ return frame.iloc[offset:stop].reset_index(drop=True)
+
+
+def _pandas_dtype(arrow_type: pa.DataType) -> pd.ArrowDtype | None:
+ """Keep categorical grouping observed-only and all other Arrow types exact."""
+ return None if pa.types.is_dictionary(arrow_type) else pd.ArrowDtype(arrow_type)
+
+
+def transform_result(
+ result: SemanticResult,
+ query: SemanticQuery,
+ decision: ReuseDecision,
+ capabilities: ContainmentCapabilities,
+) -> SemanticResult:
+ """Transform a proven-compatible cached result to the requested result."""
+ try:
+ columns: list[str] = [
+ *(dimension.name for dimension in query.dimensions),
+ *(metric.name for metric in query.metrics),
+ ]
+ if decision.mode is not ReuseMode.ROLLUP and not decision.leftover_filters:
+ offset: int = query.offset or 0
+ length: int | None = query.limit
+ table: pa.Table = result.results.select(columns).slice(offset, length)
+ return SemanticResult(requests=result.requests, results=table)
+
+ frame: pd.DataFrame = result.results.to_pandas(types_mapper=_pandas_dtype)
+ frame = _apply_leftovers(frame, decision.leftover_filters, capabilities)
+ if decision.mode is ReuseMode.ROLLUP:
+ frame = _rollup(frame, query)
+ frame = _project(frame, query)
+ frame = _slice(frame, query)
+ return SemanticResult(
+ requests=result.requests,
+ results=pa.Table.from_pandas(
+ frame,
+ schema=pa.schema(
+ [result.results.schema.field(name) for name in columns]
+ ),
+ preserve_index=False,
+ safe=True,
+ ),
+ )
+ except (
+ AttributeError,
+ KeyError,
+ TypeError,
+ ValueError,
+ OverflowError,
+ NotImplementedError,
+ pa.ArrowException,
+ ) as ex:
+ # AttributeError covers entries stored before results were normalized
+ # (``results is None``); treating them as incompatible turns a stale
+ # poisoned entry into a graceful miss instead of a request failure.
+ raise SemanticCacheTransformationError(
+ "Cached semantic result is incompatible with the requested transformation"
+ ) from ex
diff --git a/superset/semantic_layers/cache_types.py b/superset/semantic_layers/cache_types.py
new file mode 100644
index 00000000000..73256c7b1e4
--- /dev/null
+++ b/superset/semantic_layers/cache_types.py
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Dependency-neutral values shared by semantic cache policy and storage."""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+from time import time
+
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Dimension,
+ Filter,
+ Metric,
+ SemanticResult,
+)
+
+
+class ReuseMode(str, enum.Enum):
+ """Transformation needed to answer a query from a cached result."""
+
+ EXACT = "exact"
+ PROJECT = "project"
+ ROLLUP = "rollup"
+
+
+@dataclass(frozen=True)
+class PatternSemantics:
+ """Provider guarantee for SQL LIKE wildcard and escape behavior."""
+
+ escape: str
+
+ @classmethod
+ def sql_like(cls, *, escape: str) -> PatternSemantics:
+ return cls(escape=escape)
+
+
+@dataclass(frozen=True)
+class ContainmentCapabilities:
+ """Provider semantics proven equivalent for cached post-processing."""
+
+ comparisons: bool = False
+ membership: bool = False
+ nulls: bool = False
+ pattern_semantics: PatternSemantics | None = None
+
+
+SAFE_CONTAINMENT_CAPABILITIES: ContainmentCapabilities = ContainmentCapabilities()
+ROLLUP_COMPATIBLE_AGGREGATIONS: frozenset[AggregationType] = frozenset(
+ {
+ AggregationType.SUM,
+ AggregationType.COUNT,
+ AggregationType.MIN,
+ AggregationType.MAX,
+ }
+)
+
+
+@dataclass(frozen=True)
+class ReuseDecision:
+ """Eligible transformation and predicates still applied to cached rows."""
+
+ mode: ReuseMode
+ leftover_filters: frozenset[Filter]
+
+
+@dataclass(frozen=True)
+class CachedEntry:
+ """Bounded descriptor pointing to an expiring semantic result.
+
+ Instances are pickled into the data cache and outlive the code that wrote
+ them. A pickled dataclass restores its stored attributes without applying
+ defaults for fields added later, so any change to this shape must bump
+ ``cache_identity.IDENTITY_FORMAT_VERSION`` so that no bucket written by the
+ previous shape is ever unpickled by the new one.
+ """
+
+ filters: frozenset[Filter]
+ dimensions: frozenset[Dimension]
+ metrics: frozenset[Metric]
+ limit: int | None
+ offset: int
+ order_key: str
+ group_limit_key: str
+ value_key: str
+ timestamp: float = field(default_factory=time)
+ # TTL the value was stored with, so the bucket that indexes it can be kept
+ # alive at least as long; None means the backend default.
+ timeout: int | None = None
+ write_nonce: str = ""
+
+
+@dataclass(frozen=True)
+class CachedValue:
+ """Result paired with the nonce of its publishing descriptor."""
+
+ write_nonce: str
+ result: SemanticResult
+
+
+@dataclass(frozen=True)
+class CachedResultCandidate:
+ """Loaded cached value paired with its proven transformation."""
+
+ entry: CachedEntry
+ result: SemanticResult
+ decision: ReuseDecision
+
+
+@dataclass(frozen=True)
+class SemanticCacheLookupResult:
+ """Pure lookup output with deferred maintenance identities."""
+
+ candidates: tuple[CachedResultCandidate, ...]
+ missing_value_keys: frozenset[str]
+ fence_rejections: int = 0
diff --git a/superset/semantic_layers/mapper.py b/superset/semantic_layers/mapper.py
index bb1ca71c0bc..d09d92fc3ec 100644
--- a/superset/semantic_layers/mapper.py
+++ b/superset/semantic_layers/mapper.py
@@ -25,7 +25,7 @@ single dataframe.
"""
import logging
-from collections.abc import Collection
+from collections.abc import Callable, Collection
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, tzinfo
from time import time as current_time
@@ -61,10 +61,15 @@ from superset.common.utils.time_range_utils import (
get_since_until_from_time_range,
)
from superset.connectors.sqla.models import BaseDatasource
-from superset.constants import NO_TIME_RANGE
+from superset.constants import NO_TIME_RANGE, SemanticCacheStatus
from superset.exceptions import QueryObjectValidationError
from superset.models.helpers import QueryResult
from superset.result_set import stringify_extension_columns
+from superset.semantic_layers import cache as semantic_cache
+from superset.semantic_layers.cache import SemanticCacheOutcome
+from superset.semantic_layers.cache_host import build_cache_configuration
+from superset.semantic_layers.cache_policy import ContainmentCapabilities
+from superset.semantic_layers.cache_repository import ViewMeta
from superset.semantic_layers.completeness import provider_completeness
from superset.semantic_layers.members import members_by_key
from superset.superset_typing import AdhocColumn, Column
@@ -133,6 +138,46 @@ class ValidatedQueryObject(QueryObject):
metrics: list[str] | None # type: ignore[assignment]
+def _dispatch_semantic_query(
+ datasource: BaseDatasource,
+ dispatcher: Callable[[SemanticQuery], SemanticResult],
+ query: SemanticQuery,
+ *,
+ force: bool = False,
+ cacheable: bool = True,
+ cache_timeout: int | None = None,
+) -> SemanticCacheOutcome:
+ # Normalize before any store or reuse: a provider may return
+ # ``SemanticResult(results=None)`` for zero rows, and a cached ``None``
+ # table would break every later transformation of that entry.
+ def normalized_dispatcher(dispatched_query: SemanticQuery) -> SemanticResult:
+ with provider_completeness():
+ result: SemanticResult = dispatcher(dispatched_query)
+ return _coerce_empty_result(result, dispatched_query)
+
+ if not cacheable or not semantic_cache.semantic_cache_service.state.effective:
+ return semantic_cache.semantic_cache_service.execute_provider(
+ query, normalized_dispatcher
+ )
+ cache_configuration: tuple[ViewMeta, ContainmentCapabilities] | None = (
+ build_cache_configuration(datasource, cache_timeout=cache_timeout)
+ )
+ if cache_configuration is None:
+ return semantic_cache.semantic_cache_service.execute_provider(
+ query, normalized_dispatcher
+ )
+ meta: ViewMeta
+ capabilities: ContainmentCapabilities
+ meta, capabilities = cache_configuration
+ return semantic_cache.semantic_cache_service.execute(
+ meta,
+ query,
+ normalized_dispatcher,
+ capabilities=capabilities,
+ force=force,
+ )
+
+
@dataclass(frozen=True)
class _ViewMetadata:
"""One query's member observation, preserving provider iteration order."""
@@ -174,9 +219,25 @@ def get_results(query_object: QueryObject) -> QueryResult:
# Step 2: Execute the main query (first in the list)
main_query = queries[0]
- with provider_completeness():
- main_result = dispatcher(main_query)
+ main_outcome: SemanticCacheOutcome = _dispatch_semantic_query(
+ query_object.datasource,
+ dispatcher,
+ main_query,
+ force=query_object.force_query,
+ # Row-count results share the query's logical identity but not its
+ # shape: a cached table result would satisfy a row-count lookup (and
+ # vice versa) and break server-side pagination. Containment reuse is
+ # only defined over tabular results, so row-count dispatches bypass
+ # the cache entirely. Nonzero row offsets cannot be reused either;
+ # storing those pages would only evict usable first-page descriptors.
+ cacheable=not query_object.is_rowcount and not query_object.row_offset,
+ cache_timeout=query_object.cache_timeout,
+ )
+ main_result: SemanticResult = main_outcome.result
main_result = _coerce_empty_result(main_result, main_query)
+ # One flag per dispatched semantic query, so the reported provenance
+ # covers the offset queries and not only the main one.
+ cache_hits: list[bool] = [main_outcome.cache_hit]
main_df = stringify_extension_columns(main_result.results).to_pandas(
integer_object_nulls=True,
@@ -193,6 +254,7 @@ def get_results(query_object: QueryObject) -> QueryResult:
query_object,
duration,
metadata,
+ semantic_cache_status=SemanticCacheStatus.from_hits(cache_hits),
)
# Get metric names from the main query
@@ -210,8 +272,16 @@ def get_results(query_object: QueryObject) -> QueryResult:
strict=False,
):
# Execute the offset query
- with provider_completeness():
- result = dispatcher(offset_query)
+ outcome: SemanticCacheOutcome = _dispatch_semantic_query(
+ query_object.datasource,
+ dispatcher,
+ offset_query,
+ force=query_object.force_query,
+ cacheable=not query_object.is_rowcount and not query_object.row_offset,
+ cache_timeout=query_object.cache_timeout,
+ )
+ cache_hits.append(outcome.cache_hit)
+ result: SemanticResult = outcome.result
result = _coerce_empty_result(result, offset_query)
# Add this query's requests to the collection
@@ -288,6 +358,7 @@ def get_results(query_object: QueryObject) -> QueryResult:
query_object,
duration,
metadata,
+ semantic_cache_status=SemanticCacheStatus.from_hits(cache_hits),
)
@@ -322,6 +393,8 @@ def map_semantic_result_to_query_result(
query_object: ValidatedQueryObject,
duration: timedelta,
metadata: _ViewMetadata | None = None,
+ *,
+ semantic_cache_status: SemanticCacheStatus = SemanticCacheStatus.MISS,
) -> QueryResult:
"""
Convert a SemanticResult to a QueryResult.
@@ -379,6 +452,7 @@ def map_semantic_result_to_query_result(
# Time range - pass through from original query_object
from_dttm=query_object.from_dttm,
to_dttm=query_object.to_dttm,
+ semantic_cache_status=semantic_cache_status.value,
)
@@ -424,9 +498,13 @@ def _map_query_object(
# Normalize columns (may be dicts with isColumnReference=True for time-series)
dimension_names = set(all_dimensions.keys())
- normalized_columns = {
- _normalize_column(column, dimension_names) for column in query_object.columns
- }
+ normalized_column_order: list[str] = list(
+ dict.fromkeys(
+ _normalize_column(column, dimension_names)
+ for column in query_object.columns
+ )
+ )
+ normalized_columns: set[str] = set(normalized_column_order)
metrics = [all_metrics[metric] for metric in (query_object.metrics or [])]
@@ -439,7 +517,6 @@ def _map_query_object(
# to any available variant so the axis is never silently dropped; for
# every other selected column we prefer the raw variant and otherwise
# take any available variant.
- dimensions: list[Dimension] = []
seen_non_axis: dict[str, Dimension] = {}
axis_variants: list[Dimension] = []
axis_match: Dimension | None = None
@@ -455,19 +532,25 @@ def _map_query_object(
if existing is None or (existing.grain is not None and dimension.grain is None):
seen_non_axis[dimension.name] = dimension
- if axis_match is not None:
- dimensions.append(axis_match)
+ selected_dimensions: dict[str, Dimension] = dict(seen_non_axis)
+ if axis_match is not None and time_axis_column is not None:
+ selected_dimensions[time_axis_column] = axis_match
elif axis_variants:
# No variant matches the requested grain. Prefer the raw (grain=None)
# variant; otherwise pick a deterministic fallback so the axis stays
# on the query instead of being silently dropped.
raw_variant = next((v for v in axis_variants if v.grain is None), None)
- dimensions.append(
+ selected_axis: Dimension = (
raw_variant
if raw_variant is not None
else min(axis_variants, key=lambda v: v.grain.name if v.grain else "")
)
- dimensions.extend(seen_non_axis.values())
+ selected_dimensions[selected_axis.name] = selected_axis
+ dimensions: list[Dimension] = [
+ selected_dimensions[name]
+ for name in normalized_column_order
+ if name in selected_dimensions
+ ]
order = _get_order_from_query_object(query_object, all_metrics, all_dimensions)
limit = query_object.row_limit
diff --git a/superset/superset_typing.py b/superset/superset_typing.py
index c29d7814c14..94160dcda4d 100644
--- a/superset/superset_typing.py
+++ b/superset/superset_typing.py
@@ -220,9 +220,11 @@ class QueryObjectDict(TypedDict, total=False):
# Core fields from QueryObject.to_dict()
apply_fetch_values_predicate: bool
+ cache_timeout: int | None
columns: list[Column]
extras: dict[str, Any]
filter: list["QueryObjectFilterClause"]
+ force_query: bool
from_dttm: datetime | None
granularity: str | None
inner_from_dttm: datetime | None
diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot
index a7b67c15df2..f7b23706ff1 100644
--- a/superset/translations/messages.pot
+++ b/superset/translations/messages.pot
@@ -8363,6 +8363,9 @@ msgstr ""
msgid "Loaded from cache"
msgstr ""
+msgid "Loaded from semantic cache"
+msgstr ""
+
msgid "Loading"
msgstr ""
@@ -8929,6 +8932,9 @@ msgstr ""
msgid "Mixed Chart"
msgstr ""
+msgid "Mixed cache"
+msgstr ""
+
msgid "Modified"
msgstr ""
@@ -12227,6 +12233,9 @@ msgstr ""
msgid "Semantic Views"
msgstr ""
+msgid "Semantic cache"
+msgstr ""
+
msgid "Semantic layer"
msgstr ""
diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py
index d6e299d7737..3fef84fc78d 100644
--- a/superset/utils/cache_manager.py
+++ b/superset/utils/cache_manager.py
@@ -31,6 +31,9 @@ if TYPE_CHECKING:
RedisCacheBackend,
RedisSentinelCacheBackend,
)
+ from superset.semantic_layers.cache_coordination import (
+ OwnerTokenCoordinationBackend,
+ )
logger = logging.getLogger(__name__)
@@ -323,3 +326,17 @@ class CacheManager:
Returns None if DISTRIBUTED_COORDINATION_CONFIG is not configured.
"""
return self._distributed_coordination
+
+ @property
+ def semantic_cache_coordination(self) -> OwnerTokenCoordinationBackend | None:
+ """Return coordination only when it supports owner-token leases."""
+ from superset.semantic_layers.cache_coordination import (
+ OwnerTokenCoordinationBackend,
+ )
+
+ backend: RedisCacheBackend | RedisSentinelCacheBackend | None = (
+ self._distributed_coordination
+ )
+ if isinstance(backend, OwnerTokenCoordinationBackend):
+ return backend
+ return None
diff --git a/tests/integration_tests/semantic_layers/cache_coordination_tests.py b/tests/integration_tests/semantic_layers/cache_coordination_tests.py
new file mode 100644
index 00000000000..9e087dc0c8a
--- /dev/null
+++ b/tests/integration_tests/semantic_layers/cache_coordination_tests.py
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Hermetic Redis coordination tests run by the dedicated integration CI job."""
+
+import os
+import pickle # noqa: S403
+from collections.abc import Iterator
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import replace
+from time import sleep
+from typing import Any, TypeAlias
+
+import pytest
+from superset_core.semantic_layers.types import SemanticQuery
+
+from superset.coordination.cache_backend import (
+ RedisCacheBackend,
+ RedisSentinelCacheBackend,
+)
+from superset.semantic_layers.cache_coordination import (
+ SemanticCacheCoordinationSettings,
+ SemanticCacheCoordinator,
+)
+from superset.semantic_layers.cache_policy import ContainmentCapabilities
+from superset.semantic_layers.cache_repository import (
+ CachedResultCandidate,
+ MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET,
+ SemanticCacheLookupResult,
+ SemanticCacheRepository,
+)
+from tests.unit_tests.semantic_layers.conftest import (
+ build_semantic_query,
+ build_semantic_result,
+ build_view_meta,
+)
+
+CoordinationBackend: TypeAlias = RedisCacheBackend | RedisSentinelCacheBackend
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_sample_data() -> None:
+ """Override the global integration fixture; these tests require only Redis."""
+
+
+def _backends() -> list[CoordinationBackend]:
+ redis_port: int = int(os.environ.get("SEMANTIC_CACHE_REDIS_PORT", "0"))
+ sentinel_port: int = int(os.environ.get("SEMANTIC_CACHE_SENTINEL_PORT", "0"))
+ if not redis_port or not sentinel_port:
+ pytest.skip("Semantic-cache Redis integration services are not configured")
+ return [
+ RedisCacheBackend(host="127.0.0.1", port=redis_port),
+ RedisSentinelCacheBackend(
+ sentinels=[("127.0.0.1", sentinel_port)],
+ master="mymaster",
+ force_master_ip="127.0.0.1",
+ ),
+ ]
+
+
+@pytest.fixture(params=[0, 1], ids=["redis", "sentinel"])
+def coordination_backend(
+ request: pytest.FixtureRequest,
+) -> Iterator[CoordinationBackend]:
+ backends: list[CoordinationBackend] = _backends()
+ backend: CoordinationBackend = backends[request.param]
+ backend._cache.flushdb()
+ try:
+ yield backend
+ finally:
+ backend._cache.flushdb()
+
+
+def test_expired_owner_cannot_release_successor(
+ coordination_backend: CoordinationBackend,
+) -> None:
+ assert coordination_backend.acquire_owner_token("lease", "owner-a", 1)
+ sleep(1.1)
+ assert coordination_backend.acquire_owner_token("lease", "owner-b", 5)
+
+ assert coordination_backend.release_owner_token("lease", "owner-a") is False
+ assert coordination_backend.get("lease") == b"owner-b"
+ assert coordination_backend.release_owner_token("lease", "owner-b") is True
+
+
+class _PickleCache:
+ def __init__(self, backend: CoordinationBackend) -> None:
+ self._backend: CoordinationBackend = backend
+
+ def get(self, key: str) -> object | None:
+ value: bytes | None = self._backend.get(key)
+ return pickle.loads(value) if value is not None else None # noqa: S301
+
+ def has(self, key: str) -> bool:
+ return bool(self._backend.exists(key))
+
+ def set(self, key: str, value: object, timeout: int | None = None) -> bool:
+ return bool(self._backend.set(key, pickle.dumps(value), ex=timeout))
+
+ def delete(self, key: str) -> bool:
+ return bool(self._backend.delete(key))
+
+
+def test_concurrent_dashboard_workload_preserves_results_and_bound(
+ coordination_backend: CoordinationBackend,
+) -> None:
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ coordination_backend,
+ SemanticCacheCoordinationSettings(2.0, 10),
+ )
+ cache: _PickleCache = _PickleCache(coordination_backend)
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ cache,
+ coordinator,
+ )
+ queries: list[SemanticQuery] = [
+ replace(build_semantic_query(), limit=index) for index in range(1, 25)
+ ]
+
+ def execute(query: SemanticQuery) -> bool:
+ lookup: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(),
+ query,
+ ContainmentCapabilities(),
+ )
+ if lookup.candidates:
+ return lookup.candidates[0].result == build_semantic_result()
+ return repository.store(
+ build_view_meta(),
+ query,
+ build_semantic_result(),
+ )
+
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ outcomes: list[bool] = list(executor.map(execute, queries * 2))
+
+ assert all(outcomes)
+ missing_keys: list[frozenset[str]] = []
+ for query in queries[:4]:
+ candidate: CachedResultCandidate = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ ).candidates[0]
+ assert cache.delete(candidate.entry.value_key)
+ missing_keys.append(
+ repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ ).missing_value_keys
+ )
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ repairs: list[Future[Any]] = [
+ *[
+ executor.submit(
+ repository.prune_missing,
+ build_view_meta(),
+ missing,
+ )
+ for missing in missing_keys
+ ],
+ *[
+ executor.submit(
+ repository.store,
+ build_view_meta(),
+ query,
+ build_semantic_result(),
+ )
+ for query in queries[:4]
+ ],
+ *[executor.submit(execute, query) for query in queries[4:]],
+ ]
+ repair_results: list[object] = [future.result() for future in repairs]
+
+ assert len(repair_results) == 28
+ for query in queries:
+ assert repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ ).candidates
+ raw_values: list[Any] = [
+ pickle.loads(value) # noqa: S301
+ for key in coordination_backend._cache.scan_iter()
+ if (value := coordination_backend._cache.get(key)) is not None
+ ]
+ descriptor_lists: list[list[object]] = [
+ value for value in raw_values if isinstance(value, list)
+ ]
+ assert descriptor_lists
+ assert len(descriptor_lists[0]) <= MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET
diff --git a/tests/unit_tests/charts/test_chart_data_api.py b/tests/unit_tests/charts/test_chart_data_api.py
index 0f91199423a..a86d4bf75df 100644
--- a/tests/unit_tests/charts/test_chart_data_api.py
+++ b/tests/unit_tests/charts/test_chart_data_api.py
@@ -22,7 +22,7 @@ from typing import Any, TYPE_CHECKING
from unittest.mock import MagicMock, patch
import pytest
-from flask import Flask, g
+from flask import Flask, g, Response
from superset.charts.data.api import ChartDataRestApi
from superset.charts.data.dashboard_filter_context import (
@@ -458,6 +458,50 @@ def test_send_chart_response_excludes_timing_by_default(app: SupersetApp) -> Non
assert "timing" not in query_payload
+@pytest.mark.parametrize(
+ ("semantic_cache_statuses", "expected_status"),
+ [
+ (["HIT"], "HIT"),
+ (["MISS"], "MISS"),
+ ([None], "MISS"),
+ (["MIXED"], "MIXED"),
+ (["HIT", "MISS"], "MIXED"),
+ (["HIT", "MIXED"], "MIXED"),
+ (["HIT", None], "MIXED"),
+ ],
+)
+def test_send_chart_response_reports_semantic_cache_status(
+ app: SupersetApp,
+ semantic_cache_statuses: list[str | None],
+ expected_status: str,
+) -> None:
+ query_context: MagicMock = MagicMock()
+ query_context.result_type = ChartDataResultType.FULL
+ query_context.result_format = ChartDataResultFormat.JSON
+ query_results: tuple[QueryDataResult, ...] = tuple(
+ QueryDataResult(
+ payload={"semantic_cache_status": cache_status},
+ timing=_query_timing(),
+ )
+ for cache_status in semantic_cache_statuses
+ )
+ result: ChartDataExecutionResult = ChartDataExecutionResult(
+ query_context=query_context,
+ queries=query_results,
+ )
+
+ with (
+ app.test_request_context("/api/v1/chart/data"),
+ patch(
+ "superset.charts.data.api.security_manager.is_guest_user",
+ return_value=False,
+ ),
+ ):
+ response: Response = ChartDataRestApi()._send_chart_response(result)
+
+ assert response.headers["X-Superset-Semantic-Cache"] == expected_status
+
+
def test_send_chart_response_includes_opt_in_timing(app: SupersetApp) -> None:
query_payload = {"data": [{"col1": 1}], "query": "SELECT 1"}
result = _json_execution_result(query_payload)
diff --git a/tests/unit_tests/common/test_query_actions.py b/tests/unit_tests/common/test_query_actions.py
index 996f427482b..41f6ac98246 100644
--- a/tests/unit_tests/common/test_query_actions.py
+++ b/tests/unit_tests/common/test_query_actions.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import cast
+from typing import Any, cast
from unittest.mock import MagicMock, patch
+import pandas as pd
import pytest
from superset.common import query_actions
@@ -37,6 +38,34 @@ from superset.exceptions import QueryObjectValidationError
from superset.utils.core import QueryObjectFilterClause
+@pytest.mark.parametrize("cache_status", ["HIT", "MISS", "MIXED"])
+def test_results_payload_preserves_semantic_cache_provenance(cache_status: str) -> None:
+ """RESULTS filtering retains the status consumed by the response header."""
+ context: MagicMock = MagicMock()
+ query: MagicMock = MagicMock(result_type=ChartDataResultType.RESULTS)
+ context.get_data.return_value = [{"revenue": 5}]
+ payload: dict[str, Any] = {
+ "df": pd.DataFrame({"revenue": [5]}),
+ "status": QueryStatus.SUCCESS,
+ "rowcount": 1,
+ "semantic_cache_status": cache_status,
+ "applied_filter_columns": [],
+ "rejected_filter_columns": [],
+ }
+ with (
+ patch.object(query_actions, "_get_datasource", return_value=MagicMock()),
+ patch.object(query_actions, "_detect_currency", return_value=None),
+ patch.object(query_actions, "_filter_status", return_value={}),
+ patch.object(query_actions, "extract_dataframe_dtypes", return_value=[0]),
+ ):
+ result: dict[str, Any] = query_actions._materialize_full_payload(
+ context, query, payload
+ )
+ assert result["semantic_cache_status"] == cache_status
+ assert result["data"] == [{"revenue": 5}]
+ assert "df" not in result
+
+
def test_prepare_drill_detail_query_does_not_strip_filters() -> None:
"""Drill preparation keeps filters while rewriting the row query shape."""
applied_filter: QueryObjectFilterClause = {
diff --git a/tests/unit_tests/common/test_query_cache_manager.py b/tests/unit_tests/common/test_query_cache_manager.py
index 183250fa2ad..05f1fbee09f 100644
--- a/tests/unit_tests/common/test_query_cache_manager.py
+++ b/tests/unit_tests/common/test_query_cache_manager.py
@@ -55,3 +55,21 @@ class TestQueryCacheManagerGet:
assert result.is_loaded == miss_result.is_loaded
assert result.cache_value == miss_result.cache_value
assert result.status == miss_result.status
+
+ def test_result_cache_hit_does_not_replay_semantic_cache_provenance(self) -> None:
+ mock_cache: MagicMock = MagicMock()
+ mock_cache.get.return_value = {
+ "df": None,
+ "query": "provider query",
+ "semantic_cache_status": "HIT",
+ "dttm": "2026-08-04T00:00:00+00:00",
+ }
+
+ with patch(
+ "superset.common.utils.query_cache_manager._cache",
+ {CacheRegion.DEFAULT: mock_cache},
+ ):
+ result: QueryCacheManager = QueryCacheManager.get(key="some-key")
+
+ assert result.is_cached is True
+ assert result.semantic_cache_status == "MISS"
diff --git a/tests/unit_tests/common/test_query_context_processor.py b/tests/unit_tests/common/test_query_context_processor.py
index 778912fdf1e..4d1f02194c3 100644
--- a/tests/unit_tests/common/test_query_context_processor.py
+++ b/tests/unit_tests/common/test_query_context_processor.py
@@ -46,16 +46,25 @@ def mock_query_context():
def _wire_contribution_totals(mock_query_context: MagicMock) -> None:
- """Give a mock query context the real ``prepare_contribution_totals`` behavior.
+ """Give a mock query context real totals normalization and cache metadata.
``QueryContext`` owns the normalization and the processor delegates to it, so a
bare MagicMock would return a value the processor cannot unpack.
"""
+ from superset.common.query_object import QueryObject
+
mock_query_context.prepare_contribution_totals.side_effect = lambda: (
normalize_contribution_totals(
mock_query_context.queries, mock_query_context.cache_values
)
)
+ mock_query_context.custom_cache_timeout = 300
+ mock_query_context.is_async_execution = False
+ mock_query_context.force = False
+ query: QueryObject
+ for query in mock_query_context.queries:
+ assert query.datasource is not None
+ query.datasource.database.extra = "{}"
def _query_timing() -> QueryTiming:
diff --git a/tests/unit_tests/common/test_query_context_processor_timing.py b/tests/unit_tests/common/test_query_context_processor_timing.py
index 3113c7c6d3b..7f0a4ed1515 100644
--- a/tests/unit_tests/common/test_query_context_processor_timing.py
+++ b/tests/unit_tests/common/test_query_context_processor_timing.py
@@ -17,6 +17,7 @@
from unittest.mock import MagicMock, patch
import pandas as pd
+import pytest
from superset.common.chart_data_timing import QueryDataResult, QueryTiming
from superset.common.db_query_status import QueryStatus
@@ -63,6 +64,40 @@ def _processor() -> QueryContextProcessor:
return processor
+@pytest.mark.parametrize(
+ ("timeout", "force", "expected_force"),
+ [(-1, False, True), (60, True, True), (60, False, False)],
+)
+def test_contribution_totals_receive_resolved_cache_policy(
+ timeout: int, force: bool, expected_force: bool
+) -> None:
+ """Direct totals dispatch honors the same disabled/forced cache policy."""
+ processor: QueryContextProcessor = _processor()
+ main: MagicMock = _query_obj()
+ main.post_processing = [{"operation": "contribution", "options": {}}]
+ totals: MagicMock = _query_obj()
+ context: MagicMock = MagicMock()
+ context.queries = [main, totals]
+ context.get_query_result.return_value.df = pd.DataFrame({"revenue": [5.0]})
+ processor._query_context = context
+
+ def execute(query: MagicMock) -> MagicMock:
+ assert query is totals
+ assert query.cache_timeout == timeout
+ assert query.force_query is expected_force
+ return context.get_query_result.return_value
+
+ context.get_query_result.side_effect = execute
+ with (
+ patch.object(processor, "get_cache_timeout", return_value=timeout),
+ patch.object(processor, "query_cache_key", return_value="totals-key"),
+ patch.object(processor, "_resolve_forced_query", return_value=force),
+ ):
+ processor.ensure_totals_available([0], 1)
+ context.get_query_result.assert_called_once_with(totals)
+ assert main.post_processing[0]["options"]["contribution_totals"] == {"revenue": 5.0}
+
+
def test_public_projection_is_explicit_and_versioned() -> None:
assert _query_timing().as_public_dict() == {
"version": 1,
@@ -259,3 +294,47 @@ def test_get_payload_result_keeps_timing_sidecar() -> None:
assert result.queries[0].payload == query_payload
assert result.queries[0].timing.total_ns == 12_000_000
assert "timing" not in result.queries[0].payload
+
+
+@patch("superset.common.query_context_processor.QueryCacheManager")
+def test_dataframe_payload_result_resolves_force_and_timeout_onto_query_object(
+ cache_manager: MagicMock,
+) -> None:
+ """Datasources with their own caching (semantic containment) read the
+ force decision and timeout from the query object, so the processor must
+ hand them the values it resolved for the result cache."""
+ cache: MagicMock = MagicMock()
+ cache.is_loaded = True
+ cache.is_cached = True
+ cache.df = pd.DataFrame({"col1": [1]})
+ cache.cache_dttm = "2026-01-01T00:00:00"
+ cache.queried_dttm = "2026-01-01T00:00:00"
+ cache.applied_template_filters = []
+ cache.applied_filter_columns = []
+ cache.rejected_filter_columns = []
+ cache.annotation_data = {}
+ cache.error_message = None
+ cache.query = "SELECT 1"
+ cache.status = "success"
+ cache.stacktrace = None
+ cache.sql_rowcount = 1
+ cache.bq_memory_limited = False
+ cache.bq_memory_limited_row_count = 0
+ cache_manager.get.return_value = cache
+
+ processor: QueryContextProcessor = _processor()
+ query_obj: MagicMock = _query_obj()
+ query_obj.force_query = False
+ query_obj.cache_timeout = None
+ with (
+ patch.object(processor, "query_cache_key", return_value="key"),
+ patch.object(processor, "get_cache_timeout", return_value=-1),
+ ):
+ processor.get_df_payload_result(query_obj)
+
+ # A resolved ``-1`` disables caching, which the processor expresses as a
+ # forced query; both facts must reach the query object.
+ assert query_obj.force_query is True
+ assert query_obj.cache_timeout == -1
+ cache_manager.get.assert_called_once()
+ assert cache_manager.get.call_args.kwargs["force_query"] is True
diff --git a/tests/unit_tests/config_test.py b/tests/unit_tests/config_test.py
index 21999d6a165..490cd095dab 100644
--- a/tests/unit_tests/config_test.py
+++ b/tests/unit_tests/config_test.py
@@ -88,7 +88,15 @@ def test_version_history_retention_env_loads_application_config(
assert "SUPERSET_VERSION_HISTORY_RETENTION_DAYS" not in app.config
-def test_invalid_version_history_retention_env_defers_cleanup(
+def test_semantic_containment_cache_defaults_are_safe() -> None:
+ from superset import config
+
+ assert config.DEFAULT_FEATURE_FLAGS["SEMANTIC_LAYER_CONTAINMENT_CACHE"] is False
+ assert config.SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS == 1.0
+ assert config.SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS == 30
+
+
+def test_invalid_version_history_retention_env_uses_default(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
diff --git a/tests/unit_tests/coordination/test_cache_backend.py b/tests/unit_tests/coordination/test_cache_backend.py
index bbe27906721..a61e2a617a2 100644
--- a/tests/unit_tests/coordination/test_cache_backend.py
+++ b/tests/unit_tests/coordination/test_cache_backend.py
@@ -23,9 +23,47 @@ change production connection behavior (redis-py 8 defaults to RESP3 on
the wire and a 5s socket timeout).
"""
+from typing import Any
from unittest import mock
+import pytest
from pytest_mock import MockerFixture
+from redis.exceptions import ConnectionError as RedisConnectionError
+
+
+@pytest.mark.parametrize("force_master_ip", [None, "127.0.0.2"])
+def test_sentinel_omits_unset_force_master_ip(force_master_ip: str | None) -> None:
+ """Older supported redis-py clients must not receive an unknown None kwarg."""
+ from superset.coordination.cache_backend import RedisSentinelCacheBackend
+
+ with mock.patch("superset.coordination.cache_backend.Sentinel") as sentinel:
+ RedisSentinelCacheBackend(
+ sentinels=[("localhost", 26379)],
+ master="main",
+ force_master_ip=force_master_ip,
+ )
+ arguments: dict[str, Any] = sentinel.call_args.kwargs
+ if force_master_ip is None:
+ assert "force_master_ip" not in arguments
+ else:
+ assert arguments["force_master_ip"] == force_master_ip
+
+
+def test_compare_owner_and_set_uses_two_keys_and_preserves_rejection() -> None:
+ """A rejected lease comparison must never be reported as a successful SET."""
+ from superset.coordination.cache_backend import RedisCacheBackend
+
+ backend: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ client: mock.MagicMock = mock.MagicMock()
+ backend._cache = client
+ client.eval.return_value = 0
+ assert not backend.compare_owner_and_set("lease", "stale", "value", b"old", 60)
+ arguments: tuple[Any, ...] = client.eval.call_args.args
+ assert arguments[1:] == (2, "lease", "value", "stale", b"old", 60)
+ assert "redis.call('get', KEYS[1]) ~= ARGV[1]" in arguments[0]
+ client.set.assert_not_called()
+ client.eval.return_value = 1
+ assert backend.compare_owner_and_set("lease", "fresh", "value", b"new", 0)
def test_redis_cache_backend_pins_protocol_and_timeout_defaults(
@@ -215,3 +253,154 @@ def test_redis_sentinel_cache_backend_stream_helpers(mocker: MockerFixture) -> N
assert eval_args[1:] == (1, "lock", "tok") # numkeys, KEYS[1], ARGV[1]
master.eval.return_value = 0
assert backend.compare_and_delete("lock", "other") == 0
+
+
+@pytest.mark.parametrize("backend_name", ["redis", "sentinel"])
+def test_owner_token_acquire_uses_atomic_set_nx_with_ttl(
+ backend_name: str,
+) -> None:
+ from superset.coordination.cache_backend import (
+ RedisCacheBackend,
+ RedisSentinelCacheBackend,
+ )
+
+ backend_type: type[RedisCacheBackend] | type[RedisSentinelCacheBackend] = (
+ RedisCacheBackend if backend_name == "redis" else RedisSentinelCacheBackend
+ )
+ backend: RedisCacheBackend | RedisSentinelCacheBackend = object.__new__(
+ backend_type
+ )
+ cache: mock.Mock = mock.Mock()
+ cache.set.return_value = True
+ backend._cache = cache
+
+ acquired: bool = backend.acquire_owner_token("bucket", "owner-a", 15)
+
+ assert acquired is True
+ cache.set.assert_called_once_with("bucket", "owner-a", nx=True, ex=15)
+
+
+@pytest.mark.parametrize("backend_name", ["redis", "sentinel"])
+def test_owner_token_release_is_atomic_compare_and_delete(
+ backend_name: str,
+) -> None:
+ from superset.coordination.cache_backend import (
+ RedisCacheBackend,
+ RedisSentinelCacheBackend,
+ )
+
+ backend_type: type[RedisCacheBackend] | type[RedisSentinelCacheBackend] = (
+ RedisCacheBackend if backend_name == "redis" else RedisSentinelCacheBackend
+ )
+ backend: RedisCacheBackend | RedisSentinelCacheBackend = object.__new__(
+ backend_type
+ )
+ cache: mock.Mock = mock.Mock()
+ cache.eval.return_value = 0
+ backend._cache = cache
+
+ released: bool = backend.release_owner_token("bucket", "stale-owner")
+
+ assert released is False
+ script: str = cache.eval.call_args.args[0]
+ assert "redis.call('get', KEYS[1]) == ARGV[1]" in script
+ assert "redis.call('del', KEYS[1])" in script
+ assert cache.eval.call_args.args[1:] == (1, "bucket", "stale-owner")
+
+
+@pytest.mark.parametrize("backend_name", ["redis", "sentinel"])
+def test_owner_token_refresh_is_atomic_compare_and_expire(
+ backend_name: str,
+) -> None:
+ from superset.coordination.cache_backend import (
+ RedisCacheBackend,
+ RedisSentinelCacheBackend,
+ )
+
+ backend_type: type[RedisCacheBackend] | type[RedisSentinelCacheBackend] = (
+ RedisCacheBackend if backend_name == "redis" else RedisSentinelCacheBackend
+ )
+ backend: RedisCacheBackend | RedisSentinelCacheBackend = object.__new__(
+ backend_type
+ )
+ cache: mock.Mock = mock.Mock()
+ cache.eval.return_value = 1
+ backend._cache = cache
+
+ refreshed: bool = backend.refresh_owner_token("bucket", "owner-a", 15)
+
+ assert refreshed is True
+ script: str = cache.eval.call_args.args[0]
+ assert "redis.call('get', KEYS[1]) == ARGV[1]" in script
+ assert "redis.call('expire', KEYS[1], ARGV[2])" in script
+ assert cache.eval.call_args.args[1:] == (1, "bucket", "owner-a", 15)
+
+
+class _OwnerStore:
+ def __init__(self) -> None:
+ self.values: dict[str, str] = {}
+
+ def set(
+ self,
+ key: str,
+ value: str,
+ *,
+ nx: bool,
+ ex: int,
+ ) -> bool:
+ if nx and key in self.values:
+ return False
+ self.values[key] = value
+ return ex > 0
+
+ def eval(
+ self,
+ script: str,
+ key_count: int,
+ key: str,
+ owner_token: str,
+ ) -> int:
+ assert script
+ assert key_count == 1
+ if self.values.get(key) != owner_token:
+ return 0
+ del self.values[key]
+ return 1
+
+
+def test_expired_owner_cannot_release_successor_token() -> None:
+ from superset.coordination.cache_backend import RedisCacheBackend
+
+ backend: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ cache: _OwnerStore = _OwnerStore()
+ backend._cache = cache
+
+ assert backend.acquire_owner_token("bucket", "owner-a", 1)
+ assert backend.acquire_owner_token("bucket", "owner-b", 1) is False
+ del cache.values["bucket"]
+ assert backend.acquire_owner_token("bucket", "owner-b", 1)
+ assert backend.release_owner_token("bucket", "owner-a") is False
+ assert cache.values["bucket"] == "owner-b"
+ assert backend.release_owner_token("bucket", "owner-b") is True
+
+
+@pytest.mark.parametrize("operation", ["acquire", "release", "refresh"])
+def test_owner_token_backend_errors_propagate(operation: str) -> None:
+ from superset.coordination.cache_backend import RedisCacheBackend
+
+ backend: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ cache: mock.Mock = mock.Mock()
+ getattr(
+ cache, "set" if operation == "acquire" else "eval"
+ ).side_effect = RedisConnectionError("unavailable")
+ backend._cache = cache
+
+ def operation_call() -> bool:
+ if operation == "acquire":
+ return backend.acquire_owner_token("bucket", "owner", 1)
+ if operation == "refresh":
+ return backend.refresh_owner_token("bucket", "owner", 1)
+ return backend.release_owner_token("bucket", "owner")
+
+ with pytest.raises(RedisConnectionError, match="unavailable"):
+ operation_call()
diff --git a/tests/unit_tests/initialization_test.py b/tests/unit_tests/initialization_test.py
index bfe927483a6..9095e271962 100644
--- a/tests/unit_tests/initialization_test.py
+++ b/tests/unit_tests/initialization_test.py
@@ -136,6 +136,78 @@ class TestSupersetAppInitializer:
wtforms_init.assert_called_once_with()
makedirs.assert_called_once_with("/var/lib/superset", exist_ok=True)
+ @patch("superset.initialization.logger")
+ @patch("superset.initialization.stats_logger_manager")
+ @patch("superset.initialization.cache_manager")
+ @patch("superset.initialization.feature_flag_manager")
+ def test_semantic_cache_unsupported_coordination_is_nonfatal(
+ self,
+ mock_feature_flags: MagicMock,
+ mock_cache_manager: MagicMock,
+ mock_stats_manager: MagicMock,
+ mock_logger: MagicMock,
+ ) -> None:
+ mock_app: MagicMock = MagicMock()
+ mock_app.config = {
+ "SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS": 1.0,
+ "SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS": 30,
+ }
+ mock_feature_flags.is_feature_enabled.return_value = True
+ mock_cache_manager.semantic_cache_coordination = None
+ mock_stats_manager.instance.gauge.side_effect = RuntimeError(
+ "metrics unavailable"
+ )
+ initializer: SupersetAppInitializer = SupersetAppInitializer(mock_app)
+
+ initializer.configure_semantic_cache()
+
+ mock_logger.warning.assert_called_once()
+ warning: str = mock_logger.warning.call_args.args[0]
+ assert "owner-token leases" in warning
+ mock_stats_manager.instance.gauge.assert_called_once_with(
+ "semantic_cache.containment.enabled", 0.0
+ )
+ mock_stats_manager.instance.incr.assert_called_once_with(
+ "semantic_cache.containment.unsupported"
+ )
+
+ @patch("superset.initialization.logger")
+ @patch("superset.initialization.stats_logger_manager")
+ @patch("superset.initialization.cache_manager")
+ @patch("superset.initialization.feature_flag_manager")
+ def test_semantic_cache_null_data_cache_warns_and_disables(
+ self,
+ mock_feature_flags: MagicMock,
+ mock_cache_manager: MagicMock,
+ mock_stats_manager: MagicMock,
+ mock_logger: MagicMock,
+ ) -> None:
+ """The default ``NullCache`` data cache can never serve a containment
+ hit; enabling the flag on top of it must be surfaced, not silent."""
+ from flask_caching.backends.nullcache import NullCache
+
+ mock_app: MagicMock = MagicMock()
+ mock_app.config = {
+ "SEMANTIC_CACHE_COORDINATION_WAIT_SECONDS": 1.0,
+ "SEMANTIC_CACHE_COORDINATION_LEASE_SECONDS": 30,
+ }
+ mock_feature_flags.is_feature_enabled.return_value = True
+ mock_cache_manager.data_cache = NullCache()
+ mock_cache_manager.semantic_cache_coordination = MagicMock()
+ initializer: SupersetAppInitializer = SupersetAppInitializer(mock_app)
+
+ initializer.configure_semantic_cache()
+
+ mock_logger.warning.assert_called_once()
+ warning: str = mock_logger.warning.call_args.args[0]
+ assert "DATA_CACHE_CONFIG" in warning
+ mock_stats_manager.instance.gauge.assert_called_once_with(
+ "semantic_cache.containment.enabled", 0.0
+ )
+ mock_stats_manager.instance.incr.assert_called_once_with(
+ "semantic_cache.containment.unsupported"
+ )
+
@patch("superset.initialization.logger")
def test_init_app_in_ctx_calls_sync_config_to_db(self, mock_logger):
"""Test that initialization calls app.sync_config_to_db()."""
@@ -155,6 +227,7 @@ class TestSupersetAppInitializer:
patch.object(app_initializer, "configure_auth_provider"),
patch.object(app_initializer, "configure_ssh_manager"),
patch.object(app_initializer, "configure_stats_manager"),
+ patch.object(app_initializer, "configure_semantic_cache"),
patch.object(app_initializer, "init_views"),
):
app_initializer.init_app_in_ctx()
diff --git a/tests/unit_tests/queries/query_object_test.py b/tests/unit_tests/queries/query_object_test.py
index 616658f49b8..6163864d64b 100644
--- a/tests/unit_tests/queries/query_object_test.py
+++ b/tests/unit_tests/queries/query_object_test.py
@@ -65,9 +65,11 @@ def test_default_query_object_to_dict():
query_object = QueryObject(row_limit=1)
assert query_object.to_dict() == {
"apply_fetch_values_predicate": False,
+ "cache_timeout": None,
"columns": [],
"extras": {},
"filter": [],
+ "force_query": False,
"from_dttm": None,
"granularity": None,
"group_others_when_limit_reached": False,
@@ -321,6 +323,33 @@ def test_cache_key_changes_for_new_query_object_same_params():
assert query_object2.cache_key() == cache_key1
+def test_force_query_does_not_change_result_cache_identity() -> None:
+ regular: QueryObject = QueryObject(row_limit=1, force_query=False)
+ forced: QueryObject = QueryObject(row_limit=1, force_query=True)
+
+ assert regular.force_query is False
+ assert forced.force_query is True
+ assert forced.cache_key() == regular.cache_key()
+
+
+def test_force_identity_regression_kills_faulty_identity_policy() -> None:
+ class FaultyForceIdentityQueryObject(QueryObject):
+ def cache_key(self, **extra: object) -> str:
+ return f"{super().cache_key(**extra)}:{self.force_query}"
+
+ regular: FaultyForceIdentityQueryObject = FaultyForceIdentityQueryObject(
+ row_limit=1,
+ force_query=False,
+ )
+ forced: FaultyForceIdentityQueryObject = FaultyForceIdentityQueryObject(
+ row_limit=1,
+ force_query=True,
+ )
+
+ with pytest.raises(AssertionError):
+ assert forced.cache_key() == regular.cache_key()
+
+
@patch("superset.utils.cache_keys.feature_flag_manager")
def test_cache_key_cache_query_by_user_on_no_datasource(feature_flag_mock):
"""
diff --git a/tests/unit_tests/scripts/change_detector_test.py b/tests/unit_tests/scripts/change_detector_test.py
index b8fe2c1e0e1..e413cc5e77f 100644
--- a/tests/unit_tests/scripts/change_detector_test.py
+++ b/tests/unit_tests/scripts/change_detector_test.py
@@ -188,6 +188,22 @@ def test_grype_config_changes_trigger_docker_build() -> None:
)
+@pytest.mark.parametrize(
+ "path",
+ [
+ "superset/coordination/cache_backend.py",
+ "superset/utils/cache_manager.py",
+ "superset/semantic_layers/cache_repository.py",
+ ],
+)
+def test_lease_backend_changes_trigger_semantic_cache_coordination(path: str) -> None:
+ """The real-Redis coordination job must run for the owner-token lease code
+ it exercises, which lives outside the semantic_layers package."""
+ assert change_detector.detect_changes(
+ [path], change_detector.PATTERNS["semantic-layers"]
+ )
+
+
def test_detect_languages_classifies_js_outside_frontend_dir() -> None:
"""A .js file outside superset-frontend/ is still "javascript", even
though PATTERNS groups it under "python" by directory -- the gap
diff --git a/tests/unit_tests/semantic_layers/cache_coordination_test.py b/tests/unit_tests/semantic_layers/cache_coordination_test.py
new file mode 100644
index 00000000000..a86a5882464
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_coordination_test.py
@@ -0,0 +1,525 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections.abc import Callable, Iterator
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import replace
+from threading import Barrier, Event, Lock
+from unittest.mock import MagicMock
+
+import pytest
+from redis.exceptions import ConnectionError as RedisConnectionError
+from superset_core.semantic_layers.types import SemanticQuery
+
+from superset.semantic_layers.cache_coordination import (
+ SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC,
+ SemanticCacheCoordinationSettings,
+ SemanticCacheCoordinator,
+)
+from superset.semantic_layers.cache_policy import ContainmentCapabilities
+from superset.semantic_layers.cache_repository import (
+ CachedEntry,
+ SemanticCacheCoordinationError,
+ SemanticCacheLookupResult,
+ SemanticCacheRepository,
+ SemanticCacheStoreError,
+)
+from tests.unit_tests.semantic_layers.conftest import (
+ build_semantic_query,
+ build_semantic_result,
+ build_view_meta,
+)
+
+
+@pytest.mark.parametrize(
+ "wait_seconds,lease_seconds",
+ [
+ (-1.0, 10),
+ (float("inf"), 10),
+ (float("nan"), 10),
+ (0.0, 0),
+ (0.0, -1),
+ ],
+)
+def test_coordination_settings_reject_invalid_values(
+ wait_seconds: float,
+ lease_seconds: int,
+) -> None:
+ with pytest.raises(ValueError, match="Coordination"):
+ SemanticCacheCoordinationSettings(wait_seconds, lease_seconds)
+
+
+def test_coordinator_acquires_mutates_and_releases_its_token() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = True
+ operation: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 10),
+ token_factory=lambda: "owner-a",
+ )
+
+ assert coordinator.mutate("bucket", operation) is True
+ backend.acquire_owner_token.assert_called_once_with(
+ "semantic-cache-lock:bucket", "owner-a", 10
+ )
+ operation.assert_called_once()
+ backend.release_owner_token.assert_called_once_with(
+ "semantic-cache-lock:bucket", "owner-a"
+ )
+
+
+def test_coordinator_renews_lease_during_long_mutation() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = True
+ renewed: Event = Event()
+
+ def refresh(*_: object) -> bool:
+ renewed.set()
+ return True
+
+ backend.refresh_owner_token.side_effect = refresh
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 1),
+ token_factory=lambda: "owner-a",
+ )
+
+ def wait_for_renewal() -> None:
+ assert renewed.wait(timeout=1.0)
+
+ assert coordinator.mutate("bucket", wait_for_renewal) is True
+ backend.refresh_owner_token.assert_called_with(
+ "semantic-cache-lock:bucket",
+ "owner-a",
+ 1,
+ )
+
+
+def test_renewal_backend_error_fails_the_mutation_after_release() -> None:
+ # A RedisError during background lease renewal must stop the renewal
+ # thread, still release the lease, and surface as the typed coordination
+ # failure once the operation completes.
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = True
+ refresh_attempted: Event = Event()
+
+ def refresh(*_: object) -> bool:
+ refresh_attempted.set()
+ raise RedisConnectionError("connection dropped mid-lease")
+
+ backend.refresh_owner_token.side_effect = refresh
+ metric: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 1),
+ token_factory=lambda: "owner-a",
+ failure_metric=metric,
+ )
+
+ def wait_for_failed_renewal() -> None:
+ assert refresh_attempted.wait(timeout=5.0)
+
+ with pytest.raises(SemanticCacheCoordinationError, match="renewal failed"):
+ coordinator.mutate("bucket", wait_for_failed_renewal)
+ backend.release_owner_token.assert_called_once_with(
+ "semantic-cache-lock:bucket",
+ "owner-a",
+ )
+ metric.assert_called_once_with(SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC)
+
+
+def test_lost_renewal_ownership_fails_the_mutation_after_release() -> None:
+ # refresh_owner_token returning False means another owner holds the lease:
+ # the renewal thread must stop and the mutation must fail loudly rather
+ # than complete under a lease it no longer holds.
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = True
+ refresh_attempted: Event = Event()
+
+ def refresh(*_: object) -> bool:
+ refresh_attempted.set()
+ return False
+
+ backend.refresh_owner_token.side_effect = refresh
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 1),
+ token_factory=lambda: "owner-a",
+ )
+
+ def wait_for_failed_renewal() -> None:
+ assert refresh_attempted.wait(timeout=5.0)
+
+ with pytest.raises(SemanticCacheCoordinationError, match="renewal failed"):
+ coordinator.mutate("bucket", wait_for_failed_renewal)
+ backend.release_owner_token.assert_called_once_with(
+ "semantic-cache-lock:bucket",
+ "owner-a",
+ )
+
+
+def test_coordinator_wait_is_bounded_when_lease_is_busy() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = False
+ times: Iterator[float] = iter([0.0, 0.0, 0.05, 0.1])
+ sleeper: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.1, 10),
+ clock=lambda: next(times),
+ sleeper=sleeper,
+ token_factory=lambda: "owner-a",
+ )
+
+ assert coordinator.mutate("bucket", MagicMock()) is False
+ assert backend.acquire_owner_token.call_count == 3
+ assert sleeper.call_count == 2
+
+
+def test_busy_wait_spin_is_jittered_within_its_bound() -> None:
+ """Waiters released by one lease expiry must not retry in lockstep."""
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = False
+ times: Iterator[float] = iter([0.0, 0.0, 0.05, 0.1])
+ jitters: Iterator[float] = iter([0.0, 1.0])
+ sleeper: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.1, 10),
+ clock=lambda: next(times),
+ sleeper=sleeper,
+ token_factory=lambda: "owner-a",
+ jitter=lambda: next(jitters),
+ )
+
+ assert coordinator.mutate("bucket", MagicMock()) is False
+ assert [call.args[0] for call in sleeper.call_args_list] == [0.025, 0.05]
+
+
+def test_busy_lease_timeout_emits_coordination_failure_metric() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = False
+ metric: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 10),
+ failure_metric=metric,
+ )
+
+ assert coordinator.mutate("bucket", MagicMock()) is False
+ metric.assert_called_once_with(SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC)
+
+
+def test_stale_owner_release_is_reported_and_cannot_delete_successor() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = False
+ metric: MagicMock = MagicMock()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 10),
+ token_factory=lambda: "owner-a",
+ failure_metric=metric,
+ )
+
+ with pytest.raises(SemanticCacheCoordinationError, match="ownership was lost"):
+ coordinator.mutate("bucket", MagicMock())
+
+ metric.assert_called_once_with(SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC)
+
+
+def test_coordination_metric_failure_does_not_replace_typed_failure() -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = False
+ metric: MagicMock = MagicMock(side_effect=RuntimeError("metrics unavailable"))
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 10),
+ failure_metric=metric,
+ )
+
+ with pytest.raises(SemanticCacheCoordinationError, match="ownership was lost"):
+ coordinator.mutate("bucket", MagicMock())
+
+ metric.assert_called_once_with(SEMANTIC_CACHE_COORDINATION_FAILURE_METRIC)
+
+
+def test_mutation_failure_is_logged_before_release_failure(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """Lease cleanup failure must not hide the original mutation in the logs."""
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = False
+ failure: SemanticCacheStoreError = SemanticCacheStoreError("payload write failed")
+ operation: MagicMock = MagicMock(side_effect=failure)
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend, SemanticCacheCoordinationSettings(0.0, 10)
+ )
+ with pytest.raises(SemanticCacheCoordinationError, match="ownership was lost"):
+ coordinator.mutate("bucket", operation)
+ assert "Semantic cache mutation failed before lease release" in caplog.text
+ assert "payload write failed" in caplog.text
+ backend.release_owner_token.assert_called_once()
+
+
+@pytest.mark.parametrize(
+ "operation,error_match",
+ [("acquire", "acquisition"), ("release", "release")],
+)
+def test_backend_errors_become_typed_coordination_failures(
+ operation: str,
+ error_match: str,
+) -> None:
+ backend: MagicMock = MagicMock()
+ backend.acquire_owner_token.return_value = True
+ backend.release_owner_token.return_value = True
+ getattr(backend, f"{operation}_owner_token").side_effect = RedisConnectionError(
+ "unavailable"
+ )
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ backend,
+ SemanticCacheCoordinationSettings(0.0, 10),
+ )
+
+ with pytest.raises(SemanticCacheCoordinationError, match=error_match):
+ coordinator.mutate("bucket", MagicMock())
+
+
+class _ThreadOwnerBackend:
+ def __init__(self) -> None:
+ self.owner: str | None = None
+ self.lock: Lock = Lock()
+ self.acquisitions: int = 0
+
+ def acquire_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ assert key
+ assert lease_seconds > 0
+ with self.lock:
+ if self.owner is not None:
+ return False
+ self.owner = owner_token
+ self.acquisitions += 1
+ return True
+
+ def release_owner_token(self, key: str, owner_token: str) -> bool:
+ assert key
+ with self.lock:
+ if self.owner != owner_token:
+ return False
+ self.owner = None
+ return True
+
+ def refresh_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ assert key
+ assert lease_seconds > 0
+ with self.lock:
+ return self.owner == owner_token
+
+
+class _ThreadCache:
+ def __init__(self) -> None:
+ self.values: dict[str, object] = {}
+ self.lock: Lock = Lock()
+
+ def get(self, key: str) -> object | None:
+ with self.lock:
+ return self.values.get(key)
+
+ def has(self, key: str) -> bool:
+ with self.lock:
+ return key in self.values
+
+ def set(self, key: str, value: object, timeout: int | None = None) -> bool:
+ assert timeout is None or timeout > 0
+ with self.lock:
+ self.values[key] = value
+ return True
+
+ def delete(self, key: str) -> bool:
+ with self.lock:
+ return self.values.pop(key, None) is not None
+
+
+def _coordinated_repository() -> tuple[
+ SemanticCacheRepository,
+ _ThreadCache,
+ _ThreadOwnerBackend,
+]:
+ cache: _ThreadCache = _ThreadCache()
+ owner_backend: _ThreadOwnerBackend = _ThreadOwnerBackend()
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ owner_backend,
+ SemanticCacheCoordinationSettings(1.0, 10),
+ )
+ return SemanticCacheRepository(cache, coordinator), cache, owner_backend
+
+
+def test_competing_writers_preserve_every_successful_registration() -> None:
+ repository: SemanticCacheRepository
+ cache: _ThreadCache
+ owner_backend: _ThreadOwnerBackend
+ repository, cache, owner_backend = _coordinated_repository()
+
+ def store(limit: int) -> bool:
+ return repository.store(
+ build_view_meta(),
+ replace(build_semantic_query(), limit=limit),
+ build_semantic_result(),
+ )
+
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ stored: list[bool] = list(executor.map(store, range(1, 21)))
+
+ descriptors: list[CachedEntry] = next(
+ value
+ for value in cache.values.values()
+ if isinstance(value, list)
+ and all(isinstance(entry, CachedEntry) for entry in value)
+ )
+ assert all(stored)
+ assert len(descriptors) == 20
+ assert owner_backend.acquisitions == 20
+
+ acquisitions_before_lookup: int = owner_backend.acquisitions
+ lookup: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(),
+ replace(build_semantic_query(), limit=1),
+ capabilities=ContainmentCapabilities(),
+ )
+ assert lookup.candidates
+ assert owner_backend.acquisitions == acquisitions_before_lookup
+
+
+def test_overlapping_prune_and_store_preserve_fresh_registration() -> None:
+ repository: SemanticCacheRepository
+ cache: _ThreadCache
+ owner_backend: _ThreadOwnerBackend
+ repository, cache, owner_backend = _coordinated_repository()
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+ value_key: str = next(
+ key for key, value in cache.values.items() if not isinstance(value, list)
+ )
+ cache.delete(value_key)
+ missing: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ prune_future: Future[None] = executor.submit(
+ repository.prune_missing,
+ build_view_meta(),
+ missing.missing_value_keys,
+ )
+ store_future: Future[bool] = executor.submit(
+ repository.store,
+ build_view_meta(),
+ query,
+ build_semantic_result(),
+ )
+ prune_future.result()
+ assert store_future.result() is True
+
+ repeated: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ )
+ assert repeated.candidates
+
+
+class _BlindReleaseBackend(_ThreadOwnerBackend):
+ def release_owner_token(self, key: str, owner_token: str) -> bool:
+ assert key
+ assert owner_token
+ with self.lock:
+ self.owner = None
+ return True
+
+
+def test_stale_release_regression_kills_blind_release_collaborator() -> None:
+ backend: _BlindReleaseBackend = _BlindReleaseBackend()
+ assert backend.acquire_owner_token("lease", "owner-a", 1)
+ backend.owner = None
+ assert backend.acquire_owner_token("lease", "owner-b", 1)
+
+ backend.release_owner_token("lease", "owner-a")
+
+ with pytest.raises(AssertionError):
+ assert backend.owner == "owner-b"
+
+
+class _LostUpdateCache(_ThreadCache):
+ def __init__(self) -> None:
+ super().__init__()
+ self.read_barrier: Barrier = Barrier(2)
+
+ def get(self, key: str) -> object | None:
+ value: object | None = super().get(key)
+ if value is None:
+ self.read_barrier.wait(timeout=1)
+ return value
+
+
+class _NoCoordination:
+ def mutate(self, key: str, operation: Callable[[], None]) -> bool:
+ assert key
+ operation()
+ return True
+
+
+def test_competing_writer_regression_kills_lost_update_collaborator() -> None:
+ cache: _LostUpdateCache = _LostUpdateCache()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ cache,
+ _NoCoordination(),
+ )
+
+ def store(limit: int) -> bool:
+ return repository.store(
+ build_view_meta(),
+ replace(build_semantic_query(), limit=limit),
+ build_semantic_result(),
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ stored: list[bool] = list(executor.map(store, [1, 2]))
+
+ descriptors: list[CachedEntry] = next(
+ value
+ for value in cache.values.values()
+ if isinstance(value, list)
+ and all(isinstance(entry, CachedEntry) for entry in value)
+ )
+ assert all(stored)
+ with pytest.raises(AssertionError):
+ assert len(descriptors) == 2
diff --git a/tests/unit_tests/semantic_layers/cache_host_test.py b/tests/unit_tests/semantic_layers/cache_host_test.py
new file mode 100644
index 00000000000..df904bf3310
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_host_test.py
@@ -0,0 +1,369 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import MagicMock, PropertyMock
+
+import pytest
+from flask import Flask, g
+from pytest_mock import MockerFixture
+from superset_core.semantic_layers.layer import (
+ SemanticCacheCapabilities,
+ SemanticCacheExecutionContext,
+ SemanticCacheIdentityMaterial,
+ SemanticCacheResponsibility,
+ SemanticCacheScope,
+)
+
+from superset.constants import CACHE_DISABLED_TIMEOUT
+from superset.semantic_layers.cache_host import (
+ _execution_context,
+ build_cache_configuration,
+)
+from superset.semantic_layers.cache_identity import SemanticCacheIdentityFactory
+from superset.semantic_layers.cache_policy import ContainmentCapabilities
+from superset.semantic_layers.cache_repository import ViewMeta
+
+
+def _context(
+ app: Flask,
+ datasource: MagicMock,
+ user: MagicMock,
+) -> SemanticCacheExecutionContext:
+ with app.test_request_context():
+ g.user = user
+ context: SemanticCacheExecutionContext | None = _execution_context(datasource)
+ assert context is not None
+ return context
+
+
+def test_execution_context_fingerprints_all_host_security_inputs(
+ mocker: MockerFixture,
+) -> None:
+ app: Flask = Flask(__name__)
+ datasource: MagicMock = MagicMock()
+ role: MagicMock = MagicMock(id=7, name="Gamma")
+ first_user: MagicMock = MagicMock(
+ id=1,
+ username="first",
+ roles=[role],
+ guest_token={"rls": [{"clause": "tenant = 1"}]},
+ )
+ second_user: MagicMock = MagicMock(
+ id=2,
+ username="second",
+ roles=[role],
+ guest_token={"rls": [{"clause": "tenant = 1"}]},
+ )
+ rls_key: MagicMock = mocker.patch(
+ "superset.semantic_layers.cache_host.security_manager.get_rls_cache_key",
+ return_value=["region = 'GB'-group"],
+ )
+
+ baseline: SemanticCacheExecutionContext = _context(app, datasource, first_user)
+ rls_key.return_value = ["region = 'US'-group"]
+ changed_rls: SemanticCacheExecutionContext = _context(app, datasource, first_user)
+ rls_key.return_value = ["region = 'GB'-group"]
+ changed_principal: SemanticCacheExecutionContext = _context(
+ app, datasource, second_user
+ )
+ first_user.guest_token = {"rls": [{"clause": "tenant = 2"}]}
+ changed_guest_rls: SemanticCacheExecutionContext = _context(
+ app, datasource, first_user
+ )
+
+ assert (
+ len(
+ {
+ baseline.host_identity,
+ changed_rls.host_identity,
+ changed_principal.host_identity,
+ changed_guest_rls.host_identity,
+ }
+ )
+ == 4
+ )
+ assert "tenant" not in baseline.host_identity
+ assert baseline.principal_id == "1"
+ assert baseline.role_ids == ("7",)
+
+
+def test_execution_context_requires_request_and_principal() -> None:
+ app: Flask = Flask(__name__)
+ datasource: MagicMock = MagicMock()
+
+ assert _execution_context(datasource) is None
+ with app.test_request_context():
+ g.user = MagicMock(id=None, username=None)
+ assert _execution_context(datasource) is None
+
+
+@pytest.fixture(autouse=True)
+def _no_rls(mocker: MockerFixture) -> None:
+ mocker.patch(
+ "superset.semantic_layers.cache_host.security_manager.get_rls_cache_key",
+ return_value=[],
+ )
+
+
+def _datasource() -> MagicMock:
+ layer: MagicMock = MagicMock()
+ layer.semantic_cache_responsibility = SemanticCacheResponsibility.SUPERSET
+ layer.semantic_cache_scope = SemanticCacheScope.GLOBAL
+ layer.semantic_cache_capabilities = SemanticCacheCapabilities()
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"provider": "fixture"})
+ )
+ datasource: MagicMock = MagicMock()
+ datasource.semantic_layer.implementation = layer
+ datasource.implementation.metadata_cache_token = None
+ datasource.metadata_cache_token = OBSERVATION
+ datasource.result_cache_discriminator = None
+ datasource.uuid = "orders"
+ datasource.changed_on = None
+ datasource.cache_timeout = 60
+ return datasource
+
+
+@pytest.mark.parametrize(
+ "mutation",
+ [
+ "provider_responsibility",
+ "provider_identity",
+ "invalid_scope",
+ "invalid_capabilities",
+ ],
+)
+def test_cache_configuration_bypasses_invalid_provider_contract(
+ mutation: str,
+) -> None:
+ datasource: MagicMock = _datasource()
+ layer: MagicMock = datasource.semantic_layer.implementation
+ if mutation == "provider_responsibility":
+ layer.semantic_cache_responsibility = SemanticCacheResponsibility.PROVIDER
+ elif mutation == "provider_identity":
+ layer.get_semantic_cache_provider_identity.return_value = None
+ elif mutation == "invalid_scope":
+ layer.semantic_cache_scope = "unknown"
+ else:
+ layer.semantic_cache_capabilities = object()
+
+ assert build_cache_configuration(datasource) is None
+
+
+def test_context_scope_requires_host_and_provider_identity(
+ mocker: MockerFixture,
+) -> None:
+ datasource: MagicMock = _datasource()
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.semantic_cache_scope = SemanticCacheScope.EXECUTION_CONTEXT
+ execution_context: MagicMock = mocker.patch(
+ "superset.semantic_layers.cache_host._execution_context",
+ return_value=None,
+ )
+
+ assert build_cache_configuration(datasource) is None
+
+ execution_context.return_value = SemanticCacheExecutionContext(
+ "principal", ("role",), "host"
+ )
+ layer.get_semantic_cache_context_identity.return_value = None
+
+ assert build_cache_configuration(datasource) is None
+
+
+def test_cache_configuration_bypasses_disabled_caching() -> None:
+ """``cache_timeout == -1`` means no caching; a backend would read the raw
+ sentinel as "never expire", so containment must be bypassed entirely."""
+ datasource: MagicMock = _datasource()
+ datasource.cache_timeout = CACHE_DISABLED_TIMEOUT
+
+ assert build_cache_configuration(datasource) is None
+
+
+def test_cache_configuration_bypasses_disabled_caching_resolved_per_request() -> None:
+ """A chart-level or custom ``-1`` reaches containment as the resolved
+ timeout even when the datasource itself would cache, and must bypass."""
+ datasource: MagicMock = _datasource()
+ datasource.cache_timeout = 60
+
+ assert (
+ build_cache_configuration(datasource, cache_timeout=CACHE_DISABLED_TIMEOUT)
+ is None
+ )
+
+
+def test_cache_configuration_prefers_resolved_timeout() -> None:
+ """The resolved request timeout, not the datasource default, bounds how
+ long a stored containment value may be reused."""
+ datasource: MagicMock = _datasource()
+ datasource.cache_timeout = 3600
+
+ configuration: tuple[ViewMeta, ContainmentCapabilities] | None = (
+ build_cache_configuration(datasource, cache_timeout=60)
+ )
+ assert configuration is not None
+ assert configuration[0].timeout == 60
+
+ fallback: tuple[ViewMeta, ContainmentCapabilities] | None = (
+ build_cache_configuration(datasource)
+ )
+ assert fallback is not None
+ assert fallback[0].timeout == 3600
+
+
+def test_global_scope_yields_to_row_level_security(mocker: MockerFixture) -> None:
+ """Global reuse rests on the provider's guarantee that results do not vary
+ by principal; Superset RLS is a variation the provider cannot see."""
+ datasource: MagicMock = _datasource()
+ assert build_cache_configuration(datasource) is not None
+
+ mocker.patch(
+ "superset.semantic_layers.cache_host.security_manager.get_rls_cache_key",
+ return_value=["region = 'GB'-group"],
+ )
+
+ assert build_cache_configuration(datasource) is None
+
+
+def test_secret_like_identity_material_bypasses_containment(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """A provider that offers secret-like identity material must cost a bypass
+ for the view, never a failed chart; the material is not echoed."""
+ datasource: MagicMock = _datasource()
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"api_token_version": "v1-hunter2"})
+ )
+
+ with caplog.at_level("WARNING"):
+ assert build_cache_configuration(datasource) is None
+
+ assert "secret-like identity material" in caplog.text
+ assert "hunter2" not in caplog.text
+ assert "api_token_version" not in caplog.text
+
+
+# A metadata observation identity, not a secret.
+OBSERVATION: str = "observation-1"
+
+
+def _definition_identity(
+ token: object,
+ discriminator: tuple[str, str] | None = None,
+) -> object:
+ datasource: MagicMock = _datasource()
+ datasource.metadata_cache_token = token
+ datasource.result_cache_discriminator = discriminator
+ configuration: tuple[ViewMeta, ContainmentCapabilities] | None = (
+ build_cache_configuration(datasource)
+ )
+ assert configuration is not None
+ return configuration[0].definition_identity
+
+
+def test_metadata_cache_token_is_part_of_definition_identity() -> None:
+ """A metadata refresh changes the token, not ``changed_on``, so the token
+ must key containment entries or results from the old metadata are reused."""
+ legacy: object = SemanticCacheIdentityFactory.definition({"changed_on": "None"})
+ first: object = _definition_identity("observation-1")
+ second: object = _definition_identity("observation-2")
+ assert first != second
+ assert first != legacy
+ assert _definition_identity("observation-1") == first
+
+
+def test_result_discriminator_and_host_generation_both_partition_definition() -> None:
+ unversioned: object = _definition_identity(OBSERVATION)
+ first: object = _definition_identity(OBSERVATION, ("fixture", "v1"))
+ changed_version: object = _definition_identity(OBSERVATION, ("fixture", "v2"))
+ changed_provider: object = _definition_identity(OBSERVATION, ("other", "v1"))
+ changed_generation: object = _definition_identity(
+ "observation-2", ("fixture", "v1")
+ )
+
+ assert first != unversioned
+ assert changed_version != first
+ assert changed_provider != first
+ assert changed_generation != first
+
+
+def test_missing_host_metadata_token_bypasses_containment() -> None:
+ """A provider token cannot substitute for a missing host generation."""
+ datasource: MagicMock = _datasource()
+ del datasource.metadata_cache_token
+ datasource.implementation.metadata_cache_token = OBSERVATION
+ assert build_cache_configuration(datasource) is None
+
+
+@pytest.mark.parametrize("token", [None, "", 42])
+def test_invalid_host_metadata_token_bypasses_containment(token: object) -> None:
+ """Invalid host generations must never fall back to legacy reuse."""
+ datasource: MagicMock = _datasource()
+ datasource.metadata_cache_token = token
+ datasource.implementation.metadata_cache_token = OBSERVATION
+ assert build_cache_configuration(datasource) is None
+
+
+def test_unavailable_host_metadata_token_bypasses_containment(
+ mocker: MockerFixture,
+) -> None:
+ """An AttributeError from the token property is a miss, not a legacy key."""
+ datasource: MagicMock = _datasource()
+ del datasource.metadata_cache_token
+ mocker.patch.object(
+ type(datasource),
+ "metadata_cache_token",
+ new_callable=PropertyMock,
+ side_effect=AttributeError,
+ create=True,
+ )
+ assert build_cache_configuration(datasource) is None
+
+
+def test_cache_identity_does_not_consult_provider_token_or_removed_adapter(
+ mocker: MockerFixture,
+) -> None:
+ """The captured ORM host token works with legacy provider implementations."""
+ datasource: MagicMock = _datasource()
+ layer: MagicMock = datasource.semantic_layer.implementation
+ mocker.patch.object(
+ type(layer),
+ "metadata_refresh",
+ new_callable=PropertyMock,
+ side_effect=AssertionError("removed adapter accessed"),
+ create=True,
+ )
+ mocker.patch.object(
+ type(datasource.implementation),
+ "metadata_cache_token",
+ new_callable=PropertyMock,
+ side_effect=AssertionError("provider token accessed"),
+ create=True,
+ )
+ configuration: tuple[ViewMeta, ContainmentCapabilities] | None = (
+ build_cache_configuration(datasource)
+ )
+ assert configuration is not None
+ assert configuration[0].definition_identity == _definition_identity(OBSERVATION)
+
+
+def test_legacy_definition_digest_remains_pinned() -> None:
+ """Keep the independently verified pre-token identity encoding, without reuse."""
+ assert SemanticCacheIdentityFactory.definition({"changed_on": "None"}).digest == (
+ "v4:7c038bb0c633af651913f2fb9350339c70a3373e681a53db31d8cdf31e048a73"
+ )
diff --git a/tests/unit_tests/semantic_layers/cache_identity_test.py b/tests/unit_tests/semantic_layers/cache_identity_test.py
new file mode 100644
index 00000000000..71055374dc7
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_identity_test.py
@@ -0,0 +1,280 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from dataclasses import replace
+from datetime import date, datetime, time, timedelta
+
+import pyarrow as pa
+import pytest
+from superset_core.semantic_layers.types import (
+ AdhocExpression,
+ AggregationType,
+ Dimension,
+ Filter,
+ FilterValues,
+ Grain,
+ GroupLimit,
+ Metric,
+ Operator,
+ OrderDirection,
+ PredicateType,
+ SemanticQuery,
+)
+
+from superset.semantic_layers.cache_identity import (
+ semantic_dimension_key,
+ SemanticCacheIdentityFactory,
+ SemanticCacheProviderIdentity,
+ SemanticCacheScopeIdentity,
+ SemanticDefinitionIdentity,
+ SemanticViewIdentity,
+ SensitiveIdentityMaterialError,
+)
+
+
+def test_identity_is_stable_across_mapping_order() -> None:
+ first: str = SemanticCacheIdentityFactory.definition(
+ {"name": "orders", "version": 2}
+ ).digest
+ second: str = SemanticCacheIdentityFactory.definition(
+ {"version": 2, "name": "orders"}
+ ).digest
+
+ assert first == second
+ assert first.startswith("v4:")
+
+
+def test_identity_changes_with_result_affecting_material() -> None:
+ first: str = SemanticCacheIdentityFactory.provider(
+ {"provider": "dbt", "version": 1}
+ ).digest
+ second: str = SemanticCacheIdentityFactory.provider(
+ {"provider": "dbt", "version": 2}
+ ).digest
+
+ assert first != second
+
+
+def test_identity_rejects_secret_material() -> None:
+ with pytest.raises(SensitiveIdentityMaterialError):
+ SemanticCacheIdentityFactory.provider(
+ {"provider": "dbt", "api_token": "do-not-hash-raw-secrets"}
+ )
+
+
+def test_identity_rejects_nested_secret_material() -> None:
+ with pytest.raises(SensitiveIdentityMaterialError):
+ SemanticCacheIdentityFactory.provider(
+ {
+ "provider": "dbt",
+ "authentication": {"api_token": "do-not-hash-nested-raw-secrets"},
+ }
+ )
+
+
+def _query() -> SemanticQuery:
+ country: Dimension = Dimension(
+ id="orders.country",
+ name="Country",
+ type=pa.string(),
+ grain=Grain(name="day", representation="P1D"),
+ )
+ revenue: Metric = Metric(
+ id="orders.revenue",
+ name="Revenue",
+ type=pa.float64(),
+ definition="SUM(revenue)",
+ aggregation=AggregationType.SUM,
+ )
+ predicate: Filter = Filter(
+ type=PredicateType.WHERE,
+ column=country,
+ operator=Operator.IN,
+ value=frozenset({"US", "GB"}),
+ )
+ return SemanticQuery(
+ metrics=[revenue],
+ dimensions=[country],
+ filters={predicate},
+ order=[(revenue, OrderDirection.DESC)],
+ limit=10,
+ offset=2,
+ group_limit=GroupLimit(
+ dimensions=[country],
+ top=3,
+ metric=revenue,
+ ),
+ )
+
+
+def test_query_identity_is_deterministic_for_unordered_values() -> None:
+ query: SemanticQuery = _query()
+ equivalent_filter: Filter = replace(
+ next(iter(query.filters or set())),
+ value=frozenset({"GB", "US"}),
+ )
+ equivalent: SemanticQuery = replace(query, filters={equivalent_filter})
+
+ assert SemanticCacheIdentityFactory.query(query) == (
+ SemanticCacheIdentityFactory.query(equivalent)
+ )
+
+
+@pytest.mark.parametrize(
+ "changed_query",
+ [
+ replace(_query(), limit=11),
+ replace(_query(), offset=3),
+ replace(
+ _query(),
+ metrics=[replace(_query().metrics[0], aggregation=AggregationType.MAX)],
+ ),
+ replace(
+ _query(),
+ dimensions=[
+ replace(
+ _query().dimensions[0],
+ grain=Grain(name="month", representation="P1M"),
+ )
+ ],
+ ),
+ replace(_query(), order=None),
+ replace(_query(), group_limit=None),
+ ],
+)
+def test_query_identity_changes_with_result_affecting_fields(
+ changed_query: SemanticQuery,
+) -> None:
+ assert SemanticCacheIdentityFactory.query(_query()) != (
+ SemanticCacheIdentityFactory.query(changed_query)
+ )
+
+
+def test_typed_filter_values_do_not_collide() -> None:
+ query: SemanticQuery = _query()
+ predicate: Filter = next(iter(query.filters or set()))
+ numeric: SemanticQuery = replace(
+ query,
+ filters={replace(predicate, operator=Operator.EQUALS, value=1)},
+ )
+ boolean: SemanticQuery = replace(
+ query,
+ filters={replace(predicate, operator=Operator.EQUALS, value=True)},
+ )
+
+ assert SemanticCacheIdentityFactory.query(numeric) != (
+ SemanticCacheIdentityFactory.query(boolean)
+ )
+
+
+@pytest.mark.parametrize(
+ "value",
+ [
+ ("GB", "US"),
+ datetime(2026, 1, 2, 3, 4, 5),
+ date(2026, 1, 2),
+ time(3, 4, 5),
+ timedelta(days=2),
+ ],
+)
+def test_query_identity_supports_typed_scalar_values(
+ value: FilterValues | tuple[FilterValues, ...] | frozenset[FilterValues],
+) -> None:
+ query: SemanticQuery = _query()
+ predicate: Filter = next(iter(query.filters or set()))
+ changed: SemanticQuery = replace(
+ query,
+ filters={replace(predicate, operator=Operator.EQUALS, value=value)},
+ )
+
+ assert SemanticCacheIdentityFactory.query(changed)
+
+
+def test_dimension_and_order_identity_cover_all_variants() -> None:
+ raw_dimension: Dimension = replace(_query().dimensions[0], grain=None)
+ grain_dimension: Dimension = _query().dimensions[0]
+ adhoc: AdhocExpression = AdhocExpression("rank", "SUM(revenue)")
+
+ assert semantic_dimension_key(raw_dimension) == raw_dimension.id
+ assert semantic_dimension_key(grain_dimension).endswith("@P1D")
+ assert SemanticCacheIdentityFactory.order(
+ [
+ (grain_dimension, OrderDirection.ASC),
+ (adhoc, OrderDirection.DESC),
+ ]
+ )
+ assert SemanticCacheIdentityFactory.group_limit(None) == ""
+ assert SemanticCacheIdentityFactory.group_limit(_query().group_limit)
+
+
+def test_bucket_identity_includes_all_host_boundaries() -> None:
+ view: SemanticViewIdentity = SemanticViewIdentity("orders")
+ definition: SemanticDefinitionIdentity = SemanticCacheIdentityFactory.definition(
+ {"revision": 1}
+ )
+ provider: SemanticCacheProviderIdentity = SemanticCacheIdentityFactory.provider(
+ {"type": "fixture", "catalog": "one"}
+ )
+ scope: SemanticCacheScopeIdentity = SemanticCacheIdentityFactory.scope(
+ {"tenant": "one"}
+ )
+ baseline: str = SemanticCacheIdentityFactory.bucket(
+ view,
+ definition,
+ provider,
+ scope,
+ )
+
+ alternatives: list[str] = [
+ SemanticCacheIdentityFactory.bucket(
+ SemanticViewIdentity("other"), definition, provider, scope
+ ),
+ SemanticCacheIdentityFactory.bucket(
+ view,
+ SemanticCacheIdentityFactory.definition({"revision": 2}),
+ provider,
+ scope,
+ ),
+ SemanticCacheIdentityFactory.bucket(
+ view,
+ definition,
+ SemanticCacheIdentityFactory.provider(
+ {"type": "fixture", "catalog": "two"}
+ ),
+ scope,
+ ),
+ SemanticCacheIdentityFactory.bucket(
+ view,
+ definition,
+ provider,
+ SemanticCacheIdentityFactory.scope({"tenant": "two"}),
+ ),
+ ]
+
+ assert all(alternative != baseline for alternative in alternatives)
+
+
+def test_value_identity_combines_bucket_and_query() -> None:
+ bucket: str = "semantic-cache:bucket:v4:fixture"
+
+ first: str = SemanticCacheIdentityFactory.value(bucket, _query())
+ second: str = SemanticCacheIdentityFactory.value(bucket, _query())
+
+ assert first == second
+ assert first.startswith("semantic-cache:value:v4:")
diff --git a/tests/unit_tests/semantic_layers/cache_integration_test.py b/tests/unit_tests/semantic_layers/cache_integration_test.py
new file mode 100644
index 00000000000..e4378f652ad
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_integration_test.py
@@ -0,0 +1,996 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Provider-parity tests for semantic containment caching."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from dataclasses import replace
+from datetime import date, datetime
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pyarrow as pa
+import pytest
+from pytest_mock import MockerFixture
+from superset_core.semantic_layers.errors import (
+ SemanticResultCompletenessError as ProviderCompletenessError,
+)
+from superset_core.semantic_layers.layer import (
+ SemanticCacheCapabilities,
+ SemanticCacheExecutionContext,
+ SemanticCacheIdentityMaterial,
+ SemanticCacheResponsibility,
+ SemanticCacheScope,
+)
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Dimension,
+ Filter,
+ Grains,
+ Metric,
+ Operator,
+ PredicateType,
+ SemanticQuery,
+ SemanticRequest,
+ SemanticResult,
+)
+
+from superset.exceptions import (
+ QueryObjectValidationError,
+ SemanticResultCompletenessError as HostCompletenessError,
+)
+from superset.extensions import cache_manager
+from superset.models.helpers import QueryResult
+from superset.semantic_layers import cache as semantic_cache
+from superset.semantic_layers.cache import (
+ SemanticCacheOutcome,
+ SemanticCacheService,
+ SemanticCacheState,
+)
+from superset.semantic_layers.cache_policy import ContainmentCapabilities
+from superset.semantic_layers.cache_repository import (
+ SemanticCacheRepository,
+ ViewMeta,
+)
+from superset.semantic_layers.cache_types import CachedValue
+from superset.semantic_layers.mapper import (
+ get_results,
+ ValidatedQueryObject,
+)
+from superset.utils.core import QueryObjectFilterClause
+from tests.unit_tests.semantic_layers.conftest import build_view_meta
+
+# Metadata generation identities, not authentication secrets.
+HOST_GENERATION: str = "host-generation-1"
+NEXT_HOST_GENERATION: str = "host-generation-2"
+
+
+@pytest.mark.parametrize("narrow_start", [False, True])
+@pytest.mark.parametrize(
+ "grained_axis,grained_filter", [(True, True), (True, False), (False, False)]
+)
+def test_partial_month_ranges_bypass_grained_containment(
+ narrow_start: bool, grained_axis: bool, grained_filter: bool
+) -> None:
+ """Raw WHERE bounds cannot be applied to cached monthly bucket starts."""
+ raw: Dimension = Dimension("date", "date", pa.date32())
+ axis: Dimension = replace(raw, grain=Grains.MONTH) if grained_axis else raw
+ filter_column: Dimension = axis if grained_filter else raw
+ metric: Metric = Metric(
+ "revenue",
+ "revenue",
+ pa.float64(),
+ "SUM(revenue)",
+ aggregation=AggregationType.SUM,
+ )
+ start: date = date(2024, 1, 1)
+ stop: date = date(2024, 4, 1)
+ query: SemanticQuery = SemanticQuery(
+ dimensions=[axis],
+ metrics=[metric],
+ filters={
+ Filter(
+ PredicateType.WHERE,
+ filter_column,
+ Operator.GREATER_THAN_OR_EQUAL,
+ start,
+ ),
+ Filter(PredicateType.WHERE, filter_column, Operator.LESS_THAN, stop),
+ },
+ )
+ narrowed: SemanticQuery = replace(
+ query,
+ filters={
+ Filter(
+ PredicateType.WHERE,
+ filter_column,
+ Operator.GREATER_THAN_OR_EQUAL,
+ date(2024, 1, 15) if narrow_start else start,
+ ),
+ Filter(
+ PredicateType.WHERE,
+ filter_column,
+ Operator.LESS_THAN,
+ stop if narrow_start else date(2024, 3, 15),
+ ),
+ },
+ )
+ dates: list[date] = [date(2024, 1, 1), date(2024, 2, 1), date(2024, 3, 1)]
+ totals: list[float] = [100.0, 200.0, 300.0]
+ expected_dates: list[date] = dates
+ expected_totals: list[float] = (
+ [40.0, 200.0, 300.0] if narrow_start else [100.0, 200.0, 120.0]
+ )
+ if not grained_axis:
+ dates = [
+ date(2024, 1, 1),
+ date(2024, 1, 20),
+ date(2024, 2, 1),
+ date(2024, 3, 1),
+ date(2024, 3, 20),
+ ]
+ totals = [60.0, 40.0, 200.0, 120.0, 180.0]
+ expected_dates = dates[1:] if narrow_start else dates[:-1]
+ expected_totals = totals[1:] if narrow_start else totals[:-1]
+ broad: SemanticResult = SemanticResult(
+ requests=[], results=pa.table({"date": dates, "revenue": totals})
+ )
+ expected: SemanticResult = SemanticResult(
+ requests=[],
+ results=pa.table({"date": expected_dates, "revenue": expected_totals}),
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ _InMemoryCache(), _ImmediateCoordinator()
+ )
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(), repository
+ )
+ meta: ViewMeta = build_view_meta()
+ capabilities: ContainmentCapabilities = ContainmentCapabilities(comparisons=True)
+ assert not service.execute(
+ meta, query, lambda _: broad, capabilities=capabilities
+ ).cache_hit
+ provider: MagicMock = MagicMock(return_value=expected)
+ outcome: SemanticCacheOutcome = service.execute(
+ meta, narrowed, provider, capabilities=capabilities
+ )
+ assert outcome.cache_hit is not grained_axis
+ assert outcome.result.results.to_pydict() == expected.results.to_pydict()
+ if grained_axis:
+ provider.assert_called_once_with(narrowed)
+ else:
+ provider.assert_not_called()
+
+
+class _InMemoryCache:
+ def __init__(self) -> None:
+ self.store: dict[str, object] = {}
+
+ def get(self, key: str) -> object | None:
+ return self.store.get(key)
+
+ def has(self, key: str) -> bool:
+ return key in self.store
+
+ def set(self, key: str, value: object, timeout: int | None = None) -> bool:
+ self.store[key] = value
+ return True
+
+ def delete(self, key: str) -> bool:
+ return self.store.pop(key, None) is not None
+
+
+class _ImmediateCoordinator:
+ def mutate(self, key: str, operation: Callable[[], None]) -> bool:
+ operation()
+ return True
+
+
+@pytest.fixture
+def data_cache(mocker: MockerFixture) -> _InMemoryCache:
+ cache: _InMemoryCache = _InMemoryCache()
+
+ def get_data_cache(_: object) -> _InMemoryCache:
+ return cache
+
+ mocker.patch.object(
+ type(cache_manager),
+ "data_cache",
+ property(get_data_cache),
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ cache,
+ _ImmediateCoordinator(),
+ )
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ )
+ mocker.patch(
+ "superset.semantic_layers.cache.semantic_cache_service",
+ service,
+ )
+ return cache
+
+
+@pytest.fixture
+def provider() -> MagicMock:
+ country: Dimension = Dimension(
+ id="orders.country",
+ name="country",
+ type=pa.string(),
+ )
+ city: Dimension = Dimension(id="orders.city", name="city", type=pa.string())
+ revenue: Metric = Metric(
+ id="orders.revenue",
+ name="revenue",
+ type=pa.float64(),
+ definition="SUM(revenue)",
+ aggregation=AggregationType.SUM,
+ )
+ cost: Metric = Metric(
+ id="orders.cost",
+ name="cost",
+ type=pa.float64(),
+ definition="SUM(cost)",
+ aggregation=AggregationType.SUM,
+ )
+ implementation: MagicMock = MagicMock()
+ implementation.metrics = {revenue, cost}
+ implementation.dimensions = {country, city}
+ implementation.features = frozenset()
+ implementation.get_metrics.return_value = {revenue, cost}
+ implementation.get_dimensions.return_value = {country, city}
+ return implementation
+
+
+@pytest.fixture
+def datasource(provider: MagicMock) -> MagicMock:
+ semantic_view: MagicMock = MagicMock()
+ layer: MagicMock = MagicMock()
+ layer.semantic_cache_responsibility = SemanticCacheResponsibility.SUPERSET
+ layer.semantic_cache_scope = SemanticCacheScope.GLOBAL
+ layer.semantic_cache_capabilities = SemanticCacheCapabilities(
+ comparisons=True,
+ membership=True,
+ nulls=True,
+ pattern_escape="\\",
+ )
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"type": "fixture", "catalog": "one"})
+ )
+ semantic_view.metadata_cache_token = HOST_GENERATION
+ semantic_view.result_cache_discriminator = None
+ semantic_view.implementation = provider
+ semantic_view.semantic_layer.implementation = layer
+ semantic_view.uuid = "orders-view"
+ semantic_view.changed_on = datetime(2026, 1, 1)
+ semantic_view.cache_timeout = 60
+ semantic_view.fetch_values_predicate = None
+ return semantic_view
+
+
+def _result(rows: list[tuple[str, str, float]]) -> SemanticResult:
+ frame: pd.DataFrame = pd.DataFrame(
+ rows,
+ columns=["country", "city", "revenue"],
+ )
+ return SemanticResult(
+ requests=[SemanticRequest(type="SQL", definition="provider query")],
+ results=pa.Table.from_pandas(frame, preserve_index=False),
+ )
+
+
+def _query(
+ datasource: MagicMock,
+ *,
+ columns: tuple[str, ...] = ("country", "city"),
+ minimum_revenue: float | None = None,
+ country: str | None = None,
+ metrics: tuple[str, ...] = ("revenue",),
+ force_query: bool = False,
+ cache_timeout: int | None = None,
+) -> ValidatedQueryObject:
+ filters: list[QueryObjectFilterClause] = []
+ if minimum_revenue is not None:
+ filters.append(
+ QueryObjectFilterClause(
+ col="revenue",
+ op=">",
+ val=minimum_revenue,
+ )
+ )
+ if country is not None:
+ filters.append(
+ QueryObjectFilterClause(
+ col="country",
+ op="==",
+ val=country,
+ )
+ )
+ return ValidatedQueryObject(
+ datasource=datasource,
+ metrics=list(metrics),
+ columns=list(columns),
+ filters=filters,
+ force_query=force_query,
+ cache_timeout=cache_timeout,
+ )
+
+
+def test_forced_refresh_bypasses_read_and_replaces_cached_result(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ stale: SemanticResult = _result([("GB", "London", 10.0)])
+ fresh: SemanticResult = _result([("GB", "London", 20.0)])
+ provider.get_table.return_value = stale
+ initial: QueryResult = get_results(_query(datasource))
+ provider.get_table.return_value = fresh
+
+ forced: QueryResult = get_results(_query(datasource, force_query=True))
+ repeated: QueryResult = get_results(_query(datasource))
+
+ assert initial.df["revenue"].tolist() == [10.0]
+ assert forced.df["revenue"].tolist() == [20.0]
+ assert forced.semantic_cache_status == "MISS"
+ assert repeated.df["revenue"].tolist() == [20.0]
+ assert repeated.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 2
+
+
+def test_failed_forced_refresh_does_not_replace_cached_result(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ stale: SemanticResult = _result([("GB", "London", 10.0)])
+ provider.get_table.return_value = stale
+ get_results(_query(datasource))
+ provider.get_table.side_effect = RuntimeError("provider unavailable")
+
+ with pytest.raises(RuntimeError, match="provider unavailable"):
+ get_results(_query(datasource, force_query=True))
+
+ provider.get_table.side_effect = None
+ provider.get_table.return_value = _result([("GB", "London", 30.0)])
+ repeated: QueryResult = get_results(_query(datasource))
+
+ assert repeated.df["revenue"].tolist() == [10.0]
+ assert repeated.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 2
+
+
+@pytest.mark.parametrize(
+ ("failure", "expected_type"),
+ [
+ (
+ QueryObjectValidationError("provider rejected query"),
+ QueryObjectValidationError,
+ ),
+ (ProviderCompletenessError("incomplete"), HostCompletenessError),
+ ],
+ ids=["provider-rejection", "incomplete-result"],
+)
+def test_failed_provider_miss_is_not_stored_or_swallowed(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ failure: Exception,
+ expected_type: type[Exception],
+) -> None:
+ provider.get_table.side_effect = failure
+
+ with pytest.raises(expected_type):
+ get_results(_query(datasource))
+ assert data_cache.store == {}
+
+ provider.get_table.side_effect = None
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ recovered: QueryResult = get_results(_query(datasource))
+ repeated: QueryResult = get_results(_query(datasource))
+
+ assert recovered.df["revenue"].tolist() == [20.0]
+ assert recovered.semantic_cache_status == "MISS"
+ assert repeated.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 2
+
+
+def test_forced_refresh_preserves_unrelated_query_shapes(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ provider.get_table.side_effect = [
+ _result([("US", "New York", 10.0)]),
+ _result([("GB", "London", 20.0)]),
+ _result([("US", "New York", 30.0)]),
+ ]
+ get_results(_query(datasource, country="US"))
+ get_results(_query(datasource, country="GB"))
+
+ forced: QueryResult = get_results(
+ _query(datasource, country="US", force_query=True)
+ )
+ unrelated: QueryResult = get_results(_query(datasource, country="GB"))
+
+ assert forced.df["revenue"].tolist() == [30.0]
+ assert unrelated.df["revenue"].tolist() == [20.0]
+ assert unrelated.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 3
+
+
+def test_forced_store_regression_kills_noop_store_collaborator(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ mocker: MockerFixture,
+) -> None:
+ service: SemanticCacheService = semantic_cache.semantic_cache_service
+
+ class ForcedStoreNoopService:
+ state: SemanticCacheState = service.state
+
+ def execute(
+ self,
+ meta: ViewMeta,
+ query: SemanticQuery,
+ dispatcher: Callable[[SemanticQuery], SemanticResult],
+ *,
+ capabilities: ContainmentCapabilities,
+ force: bool = False,
+ ) -> SemanticCacheOutcome:
+ if force:
+ return SemanticCacheOutcome(dispatcher(query), cache_hit=False)
+ return service.execute(
+ meta,
+ query,
+ dispatcher,
+ capabilities=capabilities,
+ )
+
+ mocker.patch.object(
+ semantic_cache,
+ "semantic_cache_service",
+ ForcedStoreNoopService(),
+ )
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ get_results(_query(datasource, force_query=True))
+ repeated: QueryResult = get_results(_query(datasource))
+
+ with pytest.raises(AssertionError):
+ assert repeated.df["revenue"].tolist() == [20.0]
+
+
+def test_exact_reuse_matches_direct_provider_result(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ expected: SemanticResult = _result([("GB", "London", 10.0)])
+ provider.get_table.return_value = expected
+
+ direct_result: QueryResult = get_results(_query(datasource))
+ cached_result: QueryResult = get_results(_query(datasource))
+ direct: pd.DataFrame = direct_result.df
+ cached: pd.DataFrame = cached_result.df
+
+ pd.testing.assert_frame_equal(cached, direct)
+ assert direct_result.semantic_cache_status == "MISS"
+ assert cached_result.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 1
+
+
+def test_producer_result_discriminator_prevents_old_containment_hit(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ datasource.result_cache_discriminator = None
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ old: QueryResult = get_results(_query(datasource))
+
+ datasource.result_cache_discriminator = ("fixture", "verified-complete-v1")
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ updated: QueryResult = get_results(_query(datasource))
+ repeated: QueryResult = get_results(_query(datasource))
+
+ assert old.df["revenue"].tolist() == [10.0]
+ assert updated.df["revenue"].tolist() == [20.0]
+ assert updated.semantic_cache_status == "MISS"
+ assert repeated.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 2
+
+
+def test_leftover_filter_matches_direct_provider_result(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ broad: SemanticResult = _result([("GB", "London", 10.0), ("US", "Boston", 20.0)])
+ provider.get_table.return_value = broad
+ get_results(_query(datasource))
+
+ cached: pd.DataFrame = get_results(_query(datasource, country="US")).df
+
+ assert cached["revenue"].tolist() == [20.0]
+ assert provider.get_table.call_count == 1
+
+
+def test_projection_and_rollup_match_provider_shape(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ broad: SemanticResult = _result([("GB", "London", 10.0), ("GB", "Leeds", 5.0)])
+ provider.get_table.return_value = broad
+ get_results(_query(datasource))
+
+ cached: pd.DataFrame = get_results(_query(datasource, columns=("country",))).df
+
+ assert cached.to_dict("records") == [{"country": "GB", "revenue": 15.0}]
+ assert provider.get_table.call_count == 1
+
+
+def test_metric_projection_matches_direct_provider_result(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ frame: pd.DataFrame = pd.DataFrame(
+ [("GB", "London", 10.0, 4.0)],
+ columns=["country", "city", "revenue", "cost"],
+ )
+ broad: SemanticResult = SemanticResult(
+ requests=[SemanticRequest(type="SQL", definition="provider query")],
+ results=pa.Table.from_pandas(frame, preserve_index=False),
+ )
+ provider.get_table.return_value = broad
+ get_results(_query(datasource, metrics=("revenue", "cost")))
+
+ cached: pd.DataFrame = get_results(_query(datasource)).df
+
+ assert cached.columns.tolist() == ["country", "city", "revenue"]
+ assert provider.get_table.call_count == 1
+
+
+def test_missing_exact_value_falls_back_to_rollup_candidate(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ exact: SemanticResult = _result([("GB", "", 15.0)])
+ broad: SemanticResult = _result([("GB", "London", 10.0), ("GB", "Leeds", 5.0)])
+ provider.get_table.side_effect = [exact, broad]
+ get_results(_query(datasource, columns=("country",)))
+ get_results(_query(datasource))
+
+ value_keys: list[str] = [
+ key for key, value in data_cache.store.items() if isinstance(value, CachedValue)
+ ]
+ assert value_keys
+ data_cache.delete(value_keys[0])
+
+ cached: pd.DataFrame = get_results(_query(datasource, columns=("country",))).df
+
+ assert cached.to_dict("records") == [{"country": "GB", "revenue": 15.0}]
+ assert provider.get_table.call_count == 2
+
+
+def test_non_additive_rollup_executes_provider(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ average: Metric = next(
+ metric for metric in provider.metrics if metric.name == "revenue"
+ )
+ provider.metrics = {
+ Metric(
+ id=average.id,
+ name=average.name,
+ type=average.type,
+ definition="AVG(revenue)",
+ aggregation=AggregationType.AVG,
+ )
+ }
+ provider.get_metrics.return_value = provider.metrics
+ provider.get_table.side_effect = [
+ _result([("GB", "London", 10.0), ("GB", "Leeds", 20.0)]),
+ _result([("GB", "", 15.0)]),
+ ]
+ get_results(_query(datasource))
+
+ get_results(_query(datasource, columns=("country",)))
+
+ assert provider.get_table.call_count == 2
+
+
+def test_provider_owned_layer_bypasses_containment(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.semantic_cache_responsibility = SemanticCacheResponsibility.PROVIDER
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ first: QueryResult = get_results(_query(datasource))
+ second: QueryResult = get_results(_query(datasource))
+
+ assert first.semantic_cache_status == "MISS"
+ assert second.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 2
+
+
+def test_execution_context_identity_isolates_cached_results(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ mocker: MockerFixture,
+) -> None:
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.semantic_cache_scope = SemanticCacheScope.EXECUTION_CONTEXT
+
+ def identity_for_context(
+ context: SemanticCacheExecutionContext,
+ ) -> SemanticCacheIdentityMaterial:
+ return SemanticCacheIdentityMaterial(
+ {"principal": context.principal_id, "roles": context.role_ids}
+ )
+
+ layer.get_semantic_cache_context_identity.side_effect = identity_for_context
+ first_context: SemanticCacheExecutionContext = SemanticCacheExecutionContext(
+ "one", ("gamma",), "first-security-context"
+ )
+ second_context: SemanticCacheExecutionContext = SemanticCacheExecutionContext(
+ "one", ("gamma",), "second-security-context"
+ )
+ mocker.patch(
+ "superset.semantic_layers.cache_host._execution_context",
+ side_effect=[first_context, first_context, second_context],
+ )
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ first: QueryResult = get_results(_query(datasource))
+ repeated: QueryResult = get_results(_query(datasource))
+ isolated: QueryResult = get_results(_query(datasource))
+
+ assert first.semantic_cache_status == "MISS"
+ assert repeated.semantic_cache_status == "HIT"
+ assert isolated.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 2
+
+
+def test_global_scope_shares_cache_without_request_context_identity(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.semantic_cache_scope = SemanticCacheScope.GLOBAL
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ first: QueryResult = get_results(_query(datasource))
+ second: QueryResult = get_results(_query(datasource))
+
+ assert first.semantic_cache_status == "MISS"
+ assert second.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 1
+ layer.get_semantic_cache_context_identity.assert_not_called()
+
+
+def test_missing_execution_context_bypasses_cache(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ mocker: MockerFixture,
+) -> None:
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.semantic_cache_scope = SemanticCacheScope.EXECUTION_CONTEXT
+ mocker.patch(
+ "superset.semantic_layers.cache_host._execution_context",
+ return_value=None,
+ )
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ first: QueryResult = get_results(_query(datasource))
+ second: QueryResult = get_results(_query(datasource))
+
+ assert first.semantic_cache_status == "MISS"
+ assert second.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 2
+
+
+def test_cache_logs_do_not_include_provider_identity_material(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ sensitive_value: str = "tenant-secret-value"
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"tenant": sensitive_value})
+ )
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ with caplog.at_level(logging.DEBUG):
+ get_results(_query(datasource))
+
+ assert sensitive_value not in caplog.text
+
+
+@pytest.mark.parametrize(
+ "identity_change",
+ ["definition", "provider_type", "provider_configuration"],
+)
+def test_result_affecting_identity_change_forces_provider_fallback(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ identity_change: str,
+) -> None:
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+ get_results(_query(datasource))
+ assert provider.get_table.call_count == 1
+ layer: MagicMock = datasource.semantic_layer.implementation
+
+ if identity_change == "definition":
+ datasource.changed_on = datetime(2026, 2, 1)
+ elif identity_change == "provider_type":
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"type": "replacement", "catalog": "one"})
+ )
+ else:
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"type": "fixture", "catalog": "two"})
+ )
+
+ get_results(_query(datasource))
+
+ assert provider.get_table.call_count == 2
+
+
+def test_rowcount_dispatch_bypasses_cache_both_directions(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """A row-count dispatch must neither reuse nor seed tabular cache entries.
+
+ The cache identity derives from the SemanticQuery alone, and the dispatched
+ result kind (table vs row count) is not part of it; letting a row-count
+ request consult the cache would let it reuse table rows (breaking
+ server-side pagination), and storing its scalar result would poison later
+ table lookups.
+ """
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ rowcount_frame: pd.DataFrame = pd.DataFrame({"rowcount": [1]})
+ provider.get_row_count.return_value = SemanticResult(
+ requests=[SemanticRequest(type="SQL", definition="count query")],
+ results=pa.Table.from_pandas(rowcount_frame, preserve_index=False),
+ )
+
+ get_results(_query(datasource))
+ stored_keys: set[str] = set(data_cache.store)
+
+ rowcount_query: ValidatedQueryObject = _query(datasource)
+ rowcount_query.is_rowcount = True
+ rowcount: QueryResult = get_results(rowcount_query)
+
+ assert rowcount.df["rowcount"].tolist() == [1]
+ assert rowcount.semantic_cache_status == "MISS"
+ assert provider.get_row_count.call_count == 1
+ # The row-count dispatch neither stored anything nor disturbed the
+ # tabular entry seeded by the first query.
+ assert set(data_cache.store) == stored_keys
+ repeated: QueryResult = get_results(_query(datasource))
+ assert repeated.semantic_cache_status == "HIT"
+ assert "rowcount" not in repeated.df.columns
+ assert repeated.df["revenue"].tolist() == [10.0]
+
+
+def test_offset_pages_bypass_cache_without_evicting_first_page(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """Unreusable pagination offsets neither read nor seed containment entries."""
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+ stored: dict[str, object] = dict(data_cache.store)
+ page: ValidatedQueryObject = _query(datasource)
+ page.row_offset = 10
+ first: QueryResult = get_results(page)
+ second: QueryResult = get_results(page)
+ assert first.semantic_cache_status == second.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 3
+ assert data_cache.store == stored
+ assert get_results(_query(datasource)).semantic_cache_status == "HIT"
+
+
+def test_none_results_are_normalized_before_store(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """A ``results=None`` provider payload must never be stored raw.
+
+ Some drivers return ``None`` for zero rows. Before normalization moved
+ ahead of the store, the first request succeeded and every later cache hit
+ crashed on ``None.select(...)``.
+ """
+ provider.get_table.return_value = SemanticResult(
+ requests=[SemanticRequest(type="SQL", definition="empty query")],
+ results=None,
+ )
+
+ first: QueryResult = get_results(_query(datasource))
+ for value in data_cache.store.values():
+ if isinstance(value, CachedValue):
+ assert value.result.results is not None
+ second: QueryResult = get_results(_query(datasource))
+
+ assert first.df.empty
+ assert second.df.empty
+ assert second.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 1
+
+
+def test_poisoned_none_entry_degrades_to_miss(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """A pre-normalization cached ``results=None`` entry must miss, not crash."""
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+
+ # Simulate an entry stored by a build that predates normalization.
+ for key, value in data_cache.store.items():
+ if isinstance(value, CachedValue):
+ data_cache.store[key] = CachedValue(
+ value.write_nonce,
+ SemanticResult(requests=value.result.requests, results=None),
+ )
+
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ recovered: QueryResult = get_results(_query(datasource))
+
+ assert recovered.df["revenue"].tolist() == [20.0]
+ assert recovered.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 2
+
+
+def test_resolved_disabled_timeout_bypasses_containment_entirely(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """A chart-level or custom ``-1`` reaches the mapper as the resolved
+ timeout. Even though the datasource itself allows caching, such a request
+ must neither store into nor be served from containment."""
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+ assert data_cache.store
+
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ uncached: QueryResult = get_results(_query(datasource, cache_timeout=-1))
+ provider.get_table.return_value = _result([("GB", "London", 30.0)])
+ repeated: QueryResult = get_results(_query(datasource, cache_timeout=-1))
+ cached: QueryResult = get_results(_query(datasource))
+
+ assert uncached.df["revenue"].tolist() == [20.0]
+ assert uncached.semantic_cache_status == "MISS"
+ assert repeated.df["revenue"].tolist() == [30.0]
+ assert repeated.semantic_cache_status == "MISS"
+ # The value stored by the cacheable request is untouched by the two
+ # uncacheable ones: still the original rows, still a hit.
+ assert cached.df["revenue"].tolist() == [10.0]
+ assert cached.semantic_cache_status == "HIT"
+ assert provider.get_table.call_count == 3
+
+
+def test_resolved_timeout_bounds_stored_containment_values(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ mocker: MockerFixture,
+) -> None:
+ """Stored values expire on the request's resolved timeout (chart or custom),
+ not on the datasource default, so a chart's own retention contract holds."""
+ set_spy: MagicMock = mocker.spy(data_cache, "set")
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ get_results(_query(datasource, cache_timeout=15))
+
+ timeouts: set[int | None] = {
+ call.kwargs.get("timeout") for call in set_spy.call_args_list
+ }
+ assert timeouts == {15}
+
+
+def test_secret_like_provider_identity_never_fails_the_query(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """The identity guard fires outside the cache service's exception boundary,
+ so it must resolve to a bypass in the host rather than an error in the
+ chart request."""
+ layer: MagicMock = datasource.semantic_layer.implementation
+ layer.get_semantic_cache_provider_identity.return_value = (
+ SemanticCacheIdentityMaterial({"tenant_token": "abc"})
+ )
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+
+ first: QueryResult = get_results(_query(datasource))
+ second: QueryResult = get_results(_query(datasource))
+
+ assert first.df["revenue"].tolist() == [10.0]
+ assert first.semantic_cache_status == "MISS"
+ assert second.semantic_cache_status == "MISS"
+ assert provider.get_table.call_count == 2
+ assert not data_cache.store
+
+
+@pytest.mark.parametrize("token", [None, "", 42])
+def test_missing_metadata_token_never_reuses_containment(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+ token: object,
+) -> None:
+ """Even populated containment cannot serve a view with no host generation."""
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ first: QueryResult = get_results(_query(datasource))
+ assert first.semantic_cache_status == "MISS"
+ assert data_cache.store
+ datasource.metadata_cache_token = token
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ second: QueryResult = get_results(_query(datasource))
+ third: QueryResult = get_results(_query(datasource))
+ assert second.semantic_cache_status == third.semantic_cache_status == "MISS"
+ assert second.df.iloc[0]["revenue"] == third.df.iloc[0]["revenue"] == 20.0
+ assert provider.get_table.call_count == 3
+
+
+def test_host_generation_rotation_prevents_containment_reuse(
+ data_cache: _InMemoryCache,
+ provider: MagicMock,
+ datasource: MagicMock,
+) -> None:
+ """A legacy provider with no observation token still follows host clears."""
+ provider.metadata_cache_token = None
+ provider.get_table.return_value = _result([("GB", "London", 10.0)])
+ get_results(_query(datasource))
+ hit: QueryResult = get_results(_query(datasource))
+ assert hit.semantic_cache_status == "HIT"
+ datasource.metadata_cache_token = NEXT_HOST_GENERATION
+ provider.get_table.return_value = _result([("GB", "London", 20.0)])
+ refreshed: QueryResult = get_results(_query(datasource))
+ assert refreshed.semantic_cache_status == "MISS"
+ assert refreshed.df.iloc[0]["revenue"] == 20.0
+ assert provider.get_table.call_count == 2
diff --git a/tests/unit_tests/semantic_layers/cache_policy_test.py b/tests/unit_tests/semantic_layers/cache_policy_test.py
new file mode 100644
index 00000000000..22f62799a05
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_policy_test.py
@@ -0,0 +1,963 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+import pandas as pd
+import pyarrow as pa
+import pytest
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Dimension,
+ Filter,
+ FilterValues,
+ Grains,
+ GroupLimit,
+ Metric,
+ Operator,
+ OrderDirection,
+ PredicateType,
+ SemanticQuery,
+ SemanticRequest,
+ SemanticResult,
+)
+
+from superset.semantic_layers import cache_policy, cache_transform
+from superset.semantic_layers.cache_identity import SemanticCacheIdentityFactory
+from superset.semantic_layers.cache_policy import (
+ ContainmentCapabilities,
+ PatternSemantics,
+ ReuseDecision,
+ ReuseMode,
+)
+from superset.semantic_layers.cache_repository import CachedEntry
+
+COUNTRY: Dimension = Dimension("country", "Country", pa.string())
+REVENUE: Dimension = Dimension("revenue", "Revenue", pa.float64())
+REVENUE_METRIC: Metric = Metric(
+ id="revenue",
+ name="Revenue",
+ type=pa.float64(),
+ definition="SUM(revenue)",
+ aggregation=AggregationType.SUM,
+)
+
+
+def where(
+ dimension: Dimension,
+ operator: Operator,
+ value: FilterValues | tuple[FilterValues, ...] | frozenset[FilterValues],
+) -> Filter:
+ return Filter(PredicateType.WHERE, dimension, operator, value)
+
+
+def _candidate(
+ *,
+ cached_filters: frozenset[Filter] = frozenset(),
+ query_filters: set[Filter] | None = None,
+) -> tuple[SemanticQuery, CachedEntry]:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY],
+ filters=query_filters,
+ )
+ entry: CachedEntry = CachedEntry(
+ filters=cached_filters,
+ dimensions=frozenset({COUNTRY}),
+ metrics=frozenset({REVENUE_METRIC}),
+ limit=None,
+ offset=0,
+ order_key="",
+ group_limit_key="",
+ value_key="value",
+ )
+ return query, entry
+
+
+def test_reuse_modes_are_stable_wire_values() -> None:
+ assert [mode.value for mode in ReuseMode] == ["exact", "project", "rollup"]
+
+
+@pytest.mark.parametrize(
+ "new_filter,cached_filter,expected",
+ [
+ (
+ where(REVENUE, Operator.EQUALS, 20),
+ where(REVENUE, Operator.EQUALS, 20),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.EQUALS, 20),
+ where(REVENUE, Operator.EQUALS, 21),
+ False,
+ ),
+ (
+ where(REVENUE, Operator.GREATER_THAN, 20),
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ where(REVENUE, Operator.GREATER_THAN, 20),
+ False,
+ ),
+ (
+ where(REVENUE, Operator.GREATER_THAN_OR_EQUAL, 11),
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.GREATER_THAN_OR_EQUAL, 10),
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ False,
+ ),
+ (
+ where(REVENUE, Operator.LESS_THAN, 5),
+ where(REVENUE, Operator.LESS_THAN_OR_EQUAL, 5),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.EQUALS, 12),
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.IN, frozenset({12, 13})),
+ where(REVENUE, Operator.GREATER_THAN, 10),
+ True,
+ ),
+ (
+ where(COUNTRY, Operator.IN, frozenset({"GB", "US"})),
+ where(COUNTRY, Operator.IN, frozenset({"GB", "US", "CA"})),
+ True,
+ ),
+ (
+ where(COUNTRY, Operator.IN, frozenset({"GB", "FR"})),
+ where(COUNTRY, Operator.IN, frozenset({"GB", "US"})),
+ False,
+ ),
+ (
+ where(COUNTRY, Operator.EQUALS, "GB"),
+ where(COUNTRY, Operator.IN, frozenset({"GB", "US"})),
+ True,
+ ),
+ (
+ where(COUNTRY, Operator.IS_NULL, None),
+ where(COUNTRY, Operator.IS_NULL, None),
+ True,
+ ),
+ (
+ where(COUNTRY, Operator.IS_NOT_NULL, None),
+ where(COUNTRY, Operator.IS_NULL, None),
+ False,
+ ),
+ (
+ where(COUNTRY, Operator.EQUALS, "GB"),
+ where(COUNTRY, Operator.IS_NOT_NULL, None),
+ True,
+ ),
+ (
+ where(REVENUE, Operator.GREATER_THAN, "10"),
+ where(REVENUE, Operator.GREATER_THAN, 9),
+ False,
+ ),
+ ],
+)
+def test_filter_implication_matrix(
+ new_filter: Filter,
+ cached_filter: Filter,
+ expected: bool,
+) -> None:
+ assert cache_policy.implies(new_filter, cached_filter) is expected
+
+
+def test_membership_implication_does_not_merge_bool_and_integer_values() -> None:
+ assert (
+ cache_policy.implies(
+ where(REVENUE, Operator.EQUALS, True),
+ where(REVENUE, Operator.IN, (1, 2)),
+ )
+ is False
+ )
+
+
+def test_escaped_pattern_requires_proven_provider_semantics() -> None:
+ cached_filter: Filter = where(COUNTRY, Operator.LIKE, r"GB\_%")
+ new_filter: Filter = where(COUNTRY, Operator.EQUALS, "GB_value")
+
+ assert cache_policy.implies(new_filter, cached_filter) is False
+ assert (
+ cache_policy.implies(
+ new_filter,
+ cached_filter,
+ pattern_semantics=cache_policy.PatternSemantics.sql_like(escape="\\"),
+ )
+ is True
+ )
+
+
+@pytest.mark.parametrize("cached_value,new_value", [(1, True), (True, 1)])
+def test_equal_boolean_and_integer_filters_are_not_contained(
+ cached_value: int | bool, new_value: int | bool
+) -> None:
+ """Dataclass equality must not bypass the typed implication check."""
+ cached: Filter = where(COUNTRY, Operator.EQUALS, cached_value)
+ requested: Filter = where(COUNTRY, Operator.EQUALS, new_value)
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({cached}), query_filters={requested}
+ )
+ assert not cache_policy.implies(requested, cached)
+ assert (
+ cache_policy.select_reuse(
+ query, entry, ContainmentCapabilities(comparisons=True)
+ )
+ is None
+ )
+
+
+def test_raw_filter_cannot_target_a_shared_grained_output_name() -> None:
+ """Even a raw variant cannot filter an output also representing bucket starts."""
+ grained: Dimension = replace(COUNTRY, grain=Grains.MONTH)
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(query_filters={where(COUNTRY, Operator.EQUALS, "GB")})
+ query = replace(query, dimensions=[COUNTRY, grained])
+ entry = replace(entry, dimensions=frozenset({COUNTRY, grained}))
+ assert (
+ cache_policy.select_reuse(
+ query, entry, ContainmentCapabilities(comparisons=True)
+ )
+ is None
+ )
+
+
+def test_pattern_fallback_rejects_incompatible_escape_contract() -> None:
+ cached_filter: Filter = where(COUNTRY, Operator.LIKE, r"GB\_%")
+ new_filter: Filter = where(COUNTRY, Operator.EQUALS, "GB_value")
+
+ assert (
+ cache_policy.implies(
+ new_filter,
+ cached_filter,
+ pattern_semantics=cache_policy.PatternSemantics.sql_like(escape="!"),
+ )
+ is False
+ )
+
+
+def test_like_compilation_collapses_redundant_wildcards() -> None:
+ assert (
+ cache_policy.compile_like_pattern("%%%%%%%%value", "\\").pattern == "^.*value$"
+ )
+
+
+def test_defensive_implication_and_capability_branches_are_conservative() -> None:
+ equals: Filter = where(REVENUE, Operator.EQUALS, 20)
+ different_column: Filter = where(COUNTRY, Operator.EQUALS, "GB")
+ invalid_membership: Filter = where(REVENUE, Operator.IN, 20)
+ unsupported_cached: Filter = where(REVENUE, Operator.NOT_LIKE, "2%")
+
+ assert cache_policy.implies(equals, different_column) is False
+ assert cache_policy._implies_membership(equals, 20) is False
+ assert cache_policy._implies_membership(invalid_membership, (20,)) is False
+ assert cache_policy._range_contains(Operator.GREATER_THAN_OR_EQUAL, 10, 10)
+ assert cache_policy._range_contains(Operator.LESS_THAN, 10, 9)
+ assert cache_policy._range_contains(Operator.LESS_THAN_OR_EQUAL, 10, 10)
+ assert cache_policy._range_contains(Operator.EQUALS, 10, 10) is False
+ assert cache_policy.implies(equals, unsupported_cached) is False
+ full_capabilities: ContainmentCapabilities = ContainmentCapabilities(
+ comparisons=True,
+ membership=True,
+ nulls=True,
+ pattern_semantics=PatternSemantics.sql_like(escape="\\"),
+ )
+ assert cache_policy._supports_filter(
+ where(REVENUE, Operator.IN, (20,)), full_capabilities
+ )
+ assert cache_policy._supports_filter(
+ where(REVENUE, Operator.IS_NULL, None), full_capabilities
+ )
+ assert cache_policy._supports_filter(
+ where(REVENUE, Operator.LIKE, "2%"), full_capabilities
+ )
+ assert (
+ cache_policy._supports_filter(
+ where(REVENUE, Operator.ADHOC, "revenue > 1"),
+ full_capabilities,
+ )
+ is False
+ )
+
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate()
+ missing_metric: Metric = replace(REVENUE_METRIC, id="cost", name="Cost")
+ assert (
+ cache_policy.select_reuse(
+ replace(query, metrics=[missing_metric]),
+ entry,
+ full_capabilities,
+ )
+ is None
+ )
+
+
+@pytest.mark.parametrize(
+ "operator,value,expected",
+ [
+ (Operator.EQUALS, 1, [True, False, False]),
+ (Operator.NOT_EQUALS, 1, [False, True, False]),
+ (Operator.IN, frozenset({1, None}), [True, False, False]),
+ (Operator.NOT_IN, frozenset({1}), [False, True, False]),
+ ],
+)
+def test_sql_null_semantics_for_scalar_and_membership_masks(
+ operator: Operator,
+ value: FilterValues | frozenset[FilterValues],
+ expected: list[bool],
+) -> None:
+ series: pd.Series = pd.Series([1, 2, None])
+
+ assert cache_transform.mask_for(series, operator, value).tolist() == expected
+
+
+@pytest.mark.parametrize(
+ "operator,expected",
+ [
+ (Operator.LIKE, [True, False, False]),
+ (Operator.NOT_LIKE, [False, True, False]),
+ ],
+)
+def test_sql_null_semantics_for_pattern_masks(
+ operator: Operator,
+ expected: list[bool],
+) -> None:
+ series: pd.Series = pd.Series(["alpha", "beta", None])
+ semantics: cache_policy.PatternSemantics = cache_policy.PatternSemantics.sql_like(
+ escape="\\"
+ )
+ assert (
+ cache_transform.mask_for(
+ series, operator, "a%", pattern_semantics=semantics
+ ).tolist()
+ == expected
+ )
+
+
+def test_mask_defensive_and_null_branches() -> None:
+ series: pd.Series = pd.Series([1, None])
+
+ assert cache_transform.mask_for(series, Operator.IS_NULL, None).tolist() == [
+ False,
+ True,
+ ]
+ assert cache_transform.mask_for(series, Operator.IS_NOT_NULL, None).tolist() == [
+ True,
+ False,
+ ]
+ assert cache_transform.mask_for(series, Operator.NOT_IN, (1, None)).tolist() == [
+ False,
+ False,
+ ]
+ assert cache_transform.mask_for(series, Operator.NOT_EQUALS, None).tolist() == [
+ False,
+ False,
+ ]
+ with pytest.raises(ValueError, match="Unsupported cached filter"):
+ cache_transform.mask_for(series, Operator.ADHOC, "x")
+
+
+def test_candidate_requires_capability_for_stronger_filter_proof() -> None:
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({where(COUNTRY, Operator.IN, ("GB", "US"))}),
+ query_filters={where(COUNTRY, Operator.EQUALS, "GB")},
+ )
+
+ assert cache_policy.select_reuse(query, entry, ContainmentCapabilities()) is None
+ decision: cache_policy.ReuseDecision | None = cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True, membership=True),
+ )
+
+ assert decision is not None
+ assert decision.mode is ReuseMode.EXACT
+ assert decision.leftover_filters == frozenset(query.filters or set())
+
+
+def test_exact_filter_set_needs_no_postprocessing_capability() -> None:
+ predicate: Filter = where(COUNTRY, Operator.EQUALS, "GB")
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({predicate}),
+ query_filters={predicate},
+ )
+
+ decision: cache_policy.ReuseDecision | None = cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(),
+ )
+
+ assert decision is not None
+ assert decision.leftover_filters == frozenset()
+
+
+def test_leftover_filter_requires_cached_projection_and_capability() -> None:
+ leftover: Filter = where(COUNTRY, Operator.EQUALS, "GB")
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(query_filters={leftover})
+
+ assert cache_policy.select_reuse(query, entry, ContainmentCapabilities()) is None
+ decision: cache_policy.ReuseDecision | None = cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ unavailable_query: SemanticQuery
+ unavailable_entry: CachedEntry
+ unavailable_query, unavailable_entry = _candidate(
+ query_filters={
+ where(
+ Dimension("cost", "Cost", pa.float64()),
+ Operator.EQUALS,
+ 1,
+ )
+ }
+ )
+
+ assert decision is not None
+ assert decision.leftover_filters == frozenset({leftover})
+ assert (
+ cache_policy.select_reuse(
+ unavailable_query,
+ unavailable_entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+@pytest.mark.parametrize("predicate_type", [PredicateType.HAVING])
+def test_having_filters_must_match_exactly(predicate_type: PredicateType) -> None:
+ cached: Filter = Filter(
+ predicate_type,
+ REVENUE_METRIC,
+ Operator.GREATER_THAN,
+ 10,
+ )
+ requested: Filter = Filter(
+ predicate_type,
+ REVENUE_METRIC,
+ Operator.GREATER_THAN,
+ 20,
+ )
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({cached}),
+ query_filters={requested},
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+def test_rollup_rejects_matching_having_evaluated_at_cached_grain() -> None:
+ city: Dimension = Dimension("city", "City", pa.string())
+ having: Filter = Filter(
+ PredicateType.HAVING,
+ REVENUE_METRIC,
+ Operator.GREATER_THAN,
+ 100,
+ )
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({having}),
+ query_filters={having},
+ )
+ entry = replace(
+ entry,
+ dimensions=frozenset({COUNTRY, city}),
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+def test_leftover_dimension_cannot_be_proved_by_colliding_metric_id() -> None:
+ raw_revenue: Dimension = Dimension("revenue", "Raw revenue", pa.float64())
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ query_filters={where(raw_revenue, Operator.GREATER_THAN, 10)},
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+def test_adhoc_filters_must_match_exactly() -> None:
+ cached: Filter = Filter(PredicateType.WHERE, None, Operator.ADHOC, "x > 10")
+ requested: Filter = Filter(PredicateType.WHERE, None, Operator.ADHOC, "x > 20")
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({cached}),
+ query_filters={requested},
+ )
+
+ assert cache_policy.select_reuse(query, entry, ContainmentCapabilities()) is None
+
+
+def test_candidate_pattern_proof_requires_matching_escape_semantics() -> None:
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ cached_filters=frozenset({where(COUNTRY, Operator.LIKE, r"GB\_%")}),
+ query_filters={where(COUNTRY, Operator.EQUALS, "GB_value")},
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(
+ comparisons=True,
+ pattern_semantics=PatternSemantics.sql_like(escape="!"),
+ ),
+ )
+ is None
+ )
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(
+ comparisons=True,
+ pattern_semantics=PatternSemantics.sql_like(escape="\\"),
+ ),
+ )
+ is not None
+ )
+
+
+def test_limited_candidate_cannot_supply_leftover_filtering() -> None:
+ leftover: Filter = where(COUNTRY, Operator.EQUALS, "GB")
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(query_filters={leftover})
+ limited_query: SemanticQuery = replace(query, limit=5)
+ limited_entry: CachedEntry = replace(entry, limit=10)
+
+ assert (
+ cache_policy.select_reuse(
+ limited_query,
+ limited_entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+def test_limited_candidate_cannot_supply_rollup() -> None:
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate()
+ city: Dimension = Dimension("city", "City", pa.string())
+ limited_query: SemanticQuery = replace(query, limit=5)
+ limited_entry: CachedEntry = replace(
+ entry,
+ dimensions=frozenset({COUNTRY, city}),
+ limit=10,
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ limited_query,
+ limited_entry,
+ ContainmentCapabilities(),
+ )
+ is None
+ )
+
+
+def test_group_limit_candidate_rejects_leftover_main_query_filter() -> None:
+ group_limit: GroupLimit = GroupLimit(
+ dimensions=[COUNTRY],
+ top=10,
+ metric=REVENUE_METRIC,
+ )
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate(
+ query_filters={where(COUNTRY, Operator.EQUALS, "GB")},
+ )
+ query = replace(query, group_limit=group_limit)
+ entry = replace(
+ entry,
+ group_limit_key=SemanticCacheIdentityFactory.group_limit(group_limit),
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ query,
+ entry,
+ ContainmentCapabilities(comparisons=True),
+ )
+ is None
+ )
+
+
+def test_ordered_candidate_cannot_supply_rollup() -> None:
+ query: SemanticQuery
+ entry: CachedEntry
+ query, entry = _candidate()
+ city: Dimension = Dimension("city", "City", pa.string())
+ ordered_query: SemanticQuery = replace(
+ query,
+ order=[(REVENUE_METRIC, OrderDirection.DESC)],
+ )
+ ordered_entry: CachedEntry = replace(
+ entry,
+ dimensions=frozenset({COUNTRY, city}),
+ order_key=SemanticCacheIdentityFactory.order(ordered_query.order),
+ )
+
+ assert (
+ cache_policy.select_reuse(
+ ordered_query,
+ ordered_entry,
+ ContainmentCapabilities(),
+ )
+ is None
+ )
+
+
+def _semantic_result(frame: pd.DataFrame) -> SemanticResult:
+ return SemanticResult(
+ requests=[SemanticRequest(type="SQL", definition="provider query")],
+ results=pa.Table.from_pandas(frame, preserve_index=False),
+ )
+
+
+def test_exact_transformation_applies_leftovers_order_offset_and_limit() -> None:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY],
+ order=[(REVENUE_METRIC, OrderDirection.DESC)],
+ limit=1,
+ offset=1,
+ )
+ leftover: Filter = where(REVENUE, Operator.GREATER_THAN, 10)
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame({"Country": ["CA", "US", "GB"], "Revenue": [30.0, None, 20.0]})
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.EXACT, frozenset({leftover})),
+ ContainmentCapabilities(comparisons=True),
+ )
+
+ assert transformed.results.to_pydict() == {
+ "Country": ["GB"],
+ "Revenue": [20.0],
+ }
+ assert transformed.requests == result.requests
+
+
+def test_exact_transformation_preserves_provider_collation_order() -> None:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY],
+ order=[(COUNTRY, OrderDirection.ASC)],
+ )
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame({"Country": ["a", "B"], "Revenue": [1.0, 2.0]})
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.EXACT, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.to_pydict()["Country"] == ["a", "B"]
+
+
+def test_exact_transformation_preserves_arrow_types() -> None:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[replace(REVENUE_METRIC, type=pa.int64())],
+ dimensions=[COUNTRY],
+ )
+ result: SemanticResult = SemanticResult(
+ requests=[],
+ results=pa.table(
+ {
+ "Country": pa.array(["GB", "US"]),
+ "Revenue": pa.array([12345, None], type=pa.int64()),
+ }
+ ),
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.EXACT, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.schema == result.results.schema
+
+
+def test_project_transformation_drops_unrequested_metrics() -> None:
+ cost: Metric = Metric(
+ id="cost",
+ name="Cost",
+ type=pa.float64(),
+ definition="SUM(cost)",
+ aggregation=AggregationType.SUM,
+ )
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY],
+ )
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame({"Country": ["GB"], "Revenue": [20.0], "Cost": [5.0]})
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.PROJECT, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert cost.name not in transformed.results.column_names
+ assert transformed.results.to_pydict() == {
+ "Country": ["GB"],
+ "Revenue": [20.0],
+ }
+
+
+def test_projection_uses_requested_column_order() -> None:
+ city: Dimension = Dimension("city", "City", pa.string())
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[city, COUNTRY],
+ )
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame(
+ {
+ "Country": ["GB"],
+ "City": ["London"],
+ "Revenue": [20.0],
+ }
+ )
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.PROJECT, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.column_names == ["City", "Country", "Revenue"]
+
+
+@pytest.mark.parametrize(
+ "aggregation,values,expected",
+ [
+ (AggregationType.SUM, [2.0, 3.0], 5.0),
+ (AggregationType.COUNT, [2.0, 3.0], 5.0),
+ (AggregationType.MIN, [2.0, 3.0], 2.0),
+ (AggregationType.MAX, [2.0, 3.0], 3.0),
+ ],
+)
+def test_rollup_transformation_uses_declared_aggregation(
+ aggregation: AggregationType,
+ values: list[float],
+ expected: float,
+) -> None:
+ metric: Metric = replace(REVENUE_METRIC, aggregation=aggregation)
+ query: SemanticQuery = SemanticQuery(metrics=[metric], dimensions=[COUNTRY])
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame(
+ {
+ "Country": ["GB", "GB"],
+ "City": ["London", "Leeds"],
+ "Revenue": values,
+ }
+ )
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.ROLLUP, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.to_pydict() == {
+ "Country": ["GB"],
+ "Revenue": [expected],
+ }
+
+
+def test_sum_rollup_preserves_all_null_group() -> None:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY],
+ )
+ result: SemanticResult = _semantic_result(
+ pd.DataFrame(
+ {
+ "Country": ["GB", "GB"],
+ "City": ["London", "Leeds"],
+ "Revenue": [None, None],
+ }
+ )
+ )
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.ROLLUP, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.to_pydict() == {
+ "Country": ["GB"],
+ "Revenue": [None],
+ }
+
+
+@pytest.mark.parametrize("multiple_dimensions", [False, True])
+def test_categorical_rollup_contains_only_observed_groups(
+ multiple_dimensions: bool,
+) -> None:
+ """Dictionary dimensions must not fabricate category combinations."""
+ city: Dimension = replace(COUNTRY, id="city", name="City")
+ frame: pd.DataFrame = pd.DataFrame(
+ {
+ "Country": pd.Categorical(["GB", "GB"], categories=["GB", "US"]),
+ "City": pd.Categorical(["London", "London"], categories=["London", "NY"]),
+ "Revenue": [2.0, 3.0],
+ }
+ )
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[COUNTRY, city] if multiple_dimensions else [COUNTRY],
+ )
+ transformed: SemanticResult = cache_transform.transform_result(
+ _semantic_result(frame),
+ query,
+ ReuseDecision(ReuseMode.ROLLUP, frozenset()),
+ ContainmentCapabilities(),
+ )
+ expected: dict[str, list[str] | list[float]] = {"Country": ["GB"], "Revenue": [5.0]}
+ if multiple_dimensions:
+ expected["City"] = ["London"]
+ assert transformed.results.to_pydict() == expected
+
+
+def test_dimensionless_rollup_aggregates_all_rows() -> None:
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[],
+ )
+ result: SemanticResult = _semantic_result(pd.DataFrame({"Revenue": [2.0, 3.0]}))
+
+ transformed: SemanticResult = cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.ROLLUP, frozenset()),
+ ContainmentCapabilities(),
+ )
+
+ assert transformed.results.to_pydict() == {"Revenue": [5.0]}
+
+
+def test_transform_rejects_unproven_leftover_and_rollup() -> None:
+ result: SemanticResult = _semantic_result(pd.DataFrame({"Revenue": [2.0]}))
+ adhoc: Filter = Filter(
+ PredicateType.WHERE,
+ None,
+ Operator.ADHOC,
+ "revenue > 1",
+ )
+ query: SemanticQuery = SemanticQuery(
+ metrics=[REVENUE_METRIC],
+ dimensions=[],
+ )
+
+ with pytest.raises(
+ cache_transform.SemanticCacheTransformationError,
+ match="incompatible",
+ ):
+ cache_transform.transform_result(
+ result,
+ query,
+ ReuseDecision(ReuseMode.EXACT, frozenset({adhoc})),
+ ContainmentCapabilities(),
+ )
+
+ unsafe_metric: Metric = replace(
+ REVENUE_METRIC,
+ aggregation=AggregationType.AVG,
+ )
+ with pytest.raises(
+ cache_transform.SemanticCacheTransformationError,
+ match="incompatible",
+ ):
+ cache_transform.transform_result(
+ result,
+ replace(query, metrics=[unsafe_metric]),
+ ReuseDecision(ReuseMode.ROLLUP, frozenset()),
+ ContainmentCapabilities(),
+ )
diff --git a/tests/unit_tests/semantic_layers/cache_repository_test.py b/tests/unit_tests/semantic_layers/cache_repository_test.py
new file mode 100644
index 00000000000..cc71afbe54e
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_repository_test.py
@@ -0,0 +1,921 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections.abc import Callable, Iterator
+from contextvars import Token
+from dataclasses import fields, replace
+from typing import Any, cast
+from unittest.mock import MagicMock, patch
+
+import pytest
+from flask_caching.backends.rediscache import RedisCache
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ SemanticQuery,
+ SemanticResult,
+)
+
+from superset.coordination.cache_backend import RedisCacheBackend
+from superset.semantic_layers.cache import SafeSemanticCacheBackend
+from superset.semantic_layers.cache_coordination import (
+ SemanticCacheCoordinationSettings,
+ SemanticCacheCoordinator,
+)
+from superset.semantic_layers.cache_identity import IDENTITY_FORMAT_VERSION
+from superset.semantic_layers.cache_policy import ContainmentCapabilities, ReuseMode
+from superset.semantic_layers.cache_repository import (
+ CachedEntry,
+ MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET,
+ semantic_cache_write_fence,
+ SemanticCacheBackendError,
+ SemanticCacheCoordinationError,
+ SemanticCacheLookupResult,
+ SemanticCacheRepository,
+ SemanticCacheStoreError,
+ SemanticCacheWriteFence,
+ ViewMeta,
+)
+from superset.semantic_layers.cache_types import CachedValue
+from tests.unit_tests.semantic_layers.conftest import (
+ build_semantic_query,
+ build_semantic_result,
+ build_view_meta,
+)
+
+
+def test_descriptor_bound_is_named_and_fixed() -> None:
+ assert MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET == 128
+
+
+@pytest.mark.parametrize("owns_lease", [True, False])
+def test_distinct_redis_clients_use_immediate_nonatomic_recheck(
+ owns_lease: bool,
+) -> None:
+ """Separate-client Lua would check a lease in the wrong Redis database."""
+ coordination: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ coordination._cache = MagicMock()
+ client: MagicMock = MagicMock()
+ values: RedisCache = RedisCache(host=client)
+ safe: SafeSemanticCacheBackend = SafeSemanticCacheBackend(values)
+ check: MagicMock = MagicMock(return_value=owns_lease)
+ fence: SemanticCacheWriteFence = SemanticCacheWriteFence(
+ "lease", "owner", check, coordination
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(safe, _Coordinator())
+ token: Token[SemanticCacheWriteFence | None] = semantic_cache_write_fence.set(fence)
+ try:
+ if owns_lease:
+ repository._set("value", "fresh", 60, SemanticCacheStoreError)
+ client.set.assert_called_once()
+ else:
+ with pytest.raises(
+ SemanticCacheCoordinationError, match="ownership was lost"
+ ):
+ repository._set("value", "stale", 60, SemanticCacheStoreError)
+ client.set.assert_not_called()
+ finally:
+ semantic_cache_write_fence.reset(token)
+ check.assert_called_once_with()
+ client.eval.assert_not_called()
+ coordination._cache.eval.assert_not_called()
+
+
+class _FencedRedis:
+ """Redis command stub with a deterministic takeover before Lua executes."""
+
+ def __init__(self) -> None:
+ self.values: dict[str, bytes] = {}
+ self.timeouts: dict[str, int | None] = {}
+ self.before_set: Callable[[], None] | None = None
+ self.rejected: int = 0
+
+ def set(
+ self,
+ name: str,
+ value: bytes | str,
+ ex: int | None = None,
+ nx: bool = False,
+ **_: Any,
+ ) -> bool:
+ if nx and name in self.values:
+ return False
+ self.values[name] = value.encode() if isinstance(value, str) else value
+ self.timeouts[name] = ex
+ return True
+
+ def get(self, key: str) -> bytes | None:
+ return self.values.get(key)
+
+ def exists(self, key: str) -> bool:
+ return key in self.values
+
+ def delete(self, key: str) -> bool:
+ return self.values.pop(key, None) is not None
+
+ def eval(self, script: str, numkeys: int, *args: Any) -> int:
+ lease: str
+ owner: str
+ if numkeys == 2:
+ key: str
+ value: bytes | str
+ timeout: int
+ lease, key, owner, value, timeout = args
+ if self.before_set is not None:
+ callback: Callable[[], None] = self.before_set
+ self.before_set = None
+ callback()
+ if self.get(lease) != owner.encode():
+ self.rejected += 1
+ return 0
+ return int(self.set(key, value, ex=timeout if timeout > 0 else None))
+ rest: list[int]
+ lease, owner, *rest = args
+ if self.get(lease) != owner.encode():
+ return 0
+ if "'del'" in script:
+ return int(self.delete(lease))
+ self.timeouts[lease] = rest[0]
+ return 1
+
+
+@pytest.mark.parametrize("takeover_at", ["descriptor", "value"])
+def test_same_client_fence_rejects_paused_writer(takeover_at: str) -> None:
+ """Fresh payload, descriptor and TTL survive an old writer resuming at SET."""
+ client: _FencedRedis = _FencedRedis()
+ coordination: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ coordination._cache = client
+ values: RedisCache = RedisCache(host=client, key_prefix="data:")
+ safe: SafeSemanticCacheBackend = SafeSemanticCacheBackend(values)
+ tokens: Iterator[str] = iter(["stale", "fresh"])
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ coordination,
+ SemanticCacheCoordinationSettings(0, 10),
+ token_factory=lambda: next(tokens),
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(safe, coordinator)
+ meta: ViewMeta = build_view_meta()
+ fresh_meta: ViewMeta = replace(meta, timeout=120)
+ query: SemanticQuery = build_semantic_query()
+ stale_result: SemanticResult = build_semantic_result()
+ fresh_result: SemanticResult = replace(stale_result, requests=["fresh"])
+
+ def replace_owner() -> None:
+ lease_key: str = next(
+ key for key in client.values if key.startswith("semantic-cache-lock:")
+ )
+ client.delete(lease_key) # Expiration while the old writer was paused.
+ assert repository.store(fresh_meta, query, fresh_result)
+
+ def pause_at_value() -> None:
+ client.before_set = replace_owner
+
+ client.before_set = replace_owner if takeover_at == "descriptor" else pause_at_value
+ with pytest.raises(SemanticCacheStoreError):
+ repository.store(meta, query, stale_result)
+ assert client.rejected == 1
+ lookup: SemanticCacheLookupResult = repository.lookup(
+ fresh_meta,
+ query,
+ ContainmentCapabilities(),
+ )
+ assert lookup.candidates[0].result.requests == ["fresh"]
+ descriptor: CachedEntry = lookup.candidates[0].entry
+ assert descriptor.timeout == 120
+ assert client.timeouts[f"data:{descriptor.value_key}"] == 120
+ assert client.timeouts[f"data:{repository._bucket_key(meta)}"] == 120
+
+
+def test_foundational_builders_produce_compatible_values() -> None:
+ assert build_semantic_query().metrics[0].id == "revenue"
+ assert build_semantic_result().results.num_rows == 1
+ assert build_view_meta().view_identity.value == "fixture-view"
+
+
+class _Backend:
+ def __init__(self, *, set_succeeds: bool = True) -> None:
+ self.values: dict[str, object] = {}
+ self.timeouts: dict[str, int | None] = {}
+ self.set_succeeds: bool = set_succeeds
+
+ def get(self, key: str) -> object | None:
+ return self.values.get(key)
+
+ def has(self, key: str) -> bool:
+ return key in self.values
+
+ def set(self, key: str, value: object, timeout: int | None = None) -> bool:
+ if not self.set_succeeds:
+ return False
+ self.values[key] = value
+ self.timeouts[key] = timeout
+ return True
+
+ def delete(self, key: str) -> bool:
+ return self.values.pop(key, None) is not None
+
+
+class _Coordinator:
+ def __init__(self, *, succeeds: bool = True) -> None:
+ self.succeeds: bool = succeeds
+ self.keys: list[str] = []
+
+ def mutate(self, key: str, operation: Callable[[], None]) -> bool:
+ self.keys.append(key)
+ if not self.succeeds:
+ return False
+ operation()
+ return True
+
+
+def test_store_and_lookup_share_expiry_and_return_ranked_result() -> None:
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+ result: SemanticResult = build_semantic_result()
+
+ assert repository.store(build_view_meta(), query, result) is True
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(),
+ query,
+ ContainmentCapabilities(),
+ )
+
+ assert len(lookup_result.candidates) == 1
+ assert lookup_result.candidates[0].decision.mode is ReuseMode.EXACT
+ assert lookup_result.candidates[0].result == result
+ assert set(backend.timeouts.values()) == {300}
+ assert len(coordinator.keys) == 1
+
+
+def test_lookup_rejects_same_metric_id_with_different_semantics() -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ )
+ sum_query: SemanticQuery = replace(
+ build_semantic_query(),
+ metrics=[
+ replace(
+ build_semantic_query().metrics[0],
+ aggregation=AggregationType.SUM,
+ definition="SUM(revenue)",
+ )
+ ],
+ )
+ avg_query: SemanticQuery = replace(
+ sum_query,
+ metrics=[
+ replace(
+ sum_query.metrics[0],
+ aggregation=AggregationType.AVG,
+ definition="AVG(revenue)",
+ )
+ ],
+ )
+ repository.store(build_view_meta(), avg_query, build_semantic_result())
+
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(),
+ sum_query,
+ ContainmentCapabilities(),
+ )
+
+ assert not lookup_result.candidates
+
+
+def test_coordination_failure_raises_typed_store_error() -> None:
+ coordinator: MagicMock = MagicMock()
+ coordinator.mutate.side_effect = SemanticCacheCoordinationError("unavailable")
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ _Backend(),
+ coordinator,
+ )
+
+ with pytest.raises(SemanticCacheStoreError, match="mutation failed"):
+ repository.store(
+ build_view_meta(),
+ build_semantic_query(),
+ build_semantic_result(),
+ )
+
+
+def test_store_replaces_same_value_identity_without_duplicate_descriptor() -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ repository.store(build_view_meta(), query, build_semantic_result())
+ repository.store(build_view_meta(), query, build_semantic_result())
+
+ descriptor_lists: list[list[CachedEntry]] = [
+ value
+ for value in backend.values.values()
+ if isinstance(value, list)
+ and all(isinstance(entry, CachedEntry) for entry in value)
+ ]
+ assert len(descriptor_lists) == 1
+ assert len(descriptor_lists[0]) == 1
+
+
+def test_store_retains_only_newest_bounded_descriptors() -> None:
+ backend: _Backend = _Backend()
+ ticks: Iterator[float] = iter(
+ float(index) for index in range(MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET + 1)
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: next(ticks),
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ for limit in range(MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET + 1):
+ repository.store(
+ build_view_meta(),
+ replace(query, limit=limit),
+ build_semantic_result(),
+ )
+
+ descriptor_list: list[CachedEntry] = next(
+ value
+ for value in backend.values.values()
+ if isinstance(value, list)
+ and all(isinstance(entry, CachedEntry) for entry in value)
+ )
+ assert len(descriptor_list) == MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET
+ assert min(entry.timestamp for entry in descriptor_list) == 1.0
+ result_values: list[CachedValue] = [
+ value for value in backend.values.values() if isinstance(value, CachedValue)
+ ]
+ assert len(result_values) == MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET
+
+
+def test_store_surfaces_backend_delete_failure_as_store_error() -> None:
+ # Eviction of a bounded-out descriptor deletes its value key; a backend
+ # failure there must surface as the store's typed error, not leak the
+ # backend exception class to callers.
+ class _DeleteFailingBackend(_Backend):
+ def delete(self, key: str) -> bool:
+ raise SemanticCacheBackendError("backend down")
+
+ backend: _Backend = _DeleteFailingBackend()
+ ticks: Iterator[float] = iter(
+ float(index) for index in range(MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET + 1)
+ )
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: next(ticks),
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ for limit in range(MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET):
+ repository.store(
+ build_view_meta(),
+ replace(query, limit=limit),
+ build_semantic_result(),
+ )
+
+ with pytest.raises(SemanticCacheStoreError) as excinfo:
+ repository.store(
+ build_view_meta(),
+ replace(query, limit=MAX_SEMANTIC_CACHE_DESCRIPTORS_PER_BUCKET),
+ build_semantic_result(),
+ )
+ assert isinstance(excinfo.value.__cause__, SemanticCacheBackendError)
+
+
+def test_failed_coordination_removes_undiscoverable_value() -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(succeeds=False),
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ assert repository.store(build_view_meta(), query, build_semantic_result()) is False
+ assert not backend.values
+ assert not repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ ).candidates
+
+
+def test_store_does_not_touch_deterministic_value_when_lease_is_unavailable() -> None:
+ backend: _Backend = _Backend()
+ query: SemanticQuery = build_semantic_query()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ )
+ repository.store(build_view_meta(), query, build_semantic_result())
+ original_values: dict[str, object] = dict(backend.values)
+ losing_repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(succeeds=False),
+ )
+
+ assert (
+ losing_repository.store(
+ build_view_meta(),
+ query,
+ build_semantic_result(),
+ )
+ is False
+ )
+ assert backend.values == original_values
+
+
+def test_backend_false_write_is_a_typed_store_failure() -> None:
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ _Backend(set_succeeds=False),
+ _Coordinator(),
+ )
+
+ with pytest.raises(SemanticCacheStoreError, match="rejected set"):
+ repository.store(
+ build_view_meta(),
+ build_semantic_query(),
+ build_semantic_result(),
+ )
+
+
+def test_lookup_skips_missing_value_and_prunes_its_exact_descriptor() -> None:
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+ value_key: str = next(
+ key for key, value in backend.values.items() if isinstance(value, CachedValue)
+ )
+ backend.delete(value_key)
+ coordinator.keys.clear()
+
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ )
+ assert not lookup_result.candidates
+ assert lookup_result.missing_value_keys == frozenset({value_key})
+ assert not coordinator.keys
+ repository.prune_missing(build_view_meta(), lookup_result.missing_value_keys)
+ assert len(coordinator.keys) == 1
+ assert all(
+ not isinstance(value, list) or value == [] for value in backend.values.values()
+ )
+
+
+def test_prune_rechecks_same_key_value_registered_after_lookup() -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+ value_key: str = next(
+ key for key, value in backend.values.items() if isinstance(value, CachedValue)
+ )
+ backend.delete(value_key)
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ )
+
+ repository.store(build_view_meta(), query, build_semantic_result())
+ repository.prune_missing(build_view_meta(), lookup_result.missing_value_keys)
+ repeated: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(), query, ContainmentCapabilities()
+ )
+
+ assert repeated.candidates
+
+
+def _bucket_timeout(backend: _Backend) -> int | None:
+ bucket_key: str = next(
+ key for key, value in backend.values.items() if isinstance(value, list)
+ )
+ return backend.timeouts[bucket_key]
+
+
+def test_bucket_outlives_longest_lived_value_across_request_timeouts() -> None:
+ """Charts on one view resolve different timeouts; a short-lived store must
+ not expire the bucket while a longer-lived value it indexes is still
+ valid, or that value is orphaned until its own TTL."""
+ backend: _Backend = _Backend()
+ clock: list[float] = [10.0]
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: clock[0],
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ repository.store(
+ replace(build_view_meta(), timeout=3600),
+ replace(query, limit=1),
+ build_semantic_result(),
+ )
+ clock[0] = 20.0
+ repository.store(
+ replace(build_view_meta(), timeout=60),
+ replace(query, limit=2),
+ build_semantic_result(),
+ )
+ # 3590 s left on the first value, so the bucket keeps it discoverable.
+ assert _bucket_timeout(backend) == 3590
+
+ clock[0] = 4000.0
+ repository.store(
+ replace(build_view_meta(), timeout=60),
+ replace(query, limit=3),
+ build_semantic_result(),
+ )
+ # The first value has expired: only live values bound the bucket.
+ assert _bucket_timeout(backend) == 60
+
+
+@pytest.mark.parametrize(
+ ("timeouts", "expected"),
+ [
+ ((None, None), 0),
+ ((None, 300), 0),
+ ((300, None), 0),
+ ((300, 0), 0),
+ ],
+)
+def test_bucket_timeout_follows_backend_conventions(
+ timeouts: tuple[int | None, ...],
+ expected: int | None,
+) -> None:
+ """Unknown default TTLs cannot be shortened by an explicit bucket TTL."""
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+
+ for limit, timeout in enumerate(timeouts):
+ repository.store(
+ replace(build_view_meta(), timeout=timeout),
+ replace(query, limit=limit),
+ build_semantic_result(),
+ )
+
+ assert _bucket_timeout(backend) == expected
+
+
+def test_prune_keeps_bucket_alive_for_surviving_values() -> None:
+ backend: _Backend = _Backend()
+ clock: list[float] = [10.0]
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: clock[0],
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(
+ replace(build_view_meta(), timeout=3600),
+ replace(query, limit=1),
+ build_semantic_result(),
+ )
+ repository.store(
+ replace(build_view_meta(), timeout=60),
+ replace(query, limit=2),
+ build_semantic_result(),
+ )
+ descriptors: list[CachedEntry] = next(
+ value for value in backend.values.values() if isinstance(value, list)
+ )
+ short_lived: CachedEntry = next(
+ entry for entry in descriptors if entry.timeout == 60
+ )
+ backend.values.pop(short_lived.value_key)
+
+ clock[0] = 100.0
+ repository.prune_missing(
+ replace(build_view_meta(), timeout=60),
+ frozenset({short_lived.value_key}),
+ )
+
+ assert _bucket_timeout(backend) == 3510
+
+
+def test_store_skips_identical_value_another_request_already_registered() -> None:
+ """A herd of identical misses should contend for the lease once: once the
+ value is registered, later non-forced stores are redundant."""
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+ first: SemanticResult = build_semantic_result()
+
+ assert repository.store(build_view_meta(), query, first) is True
+ assert repository.store(build_view_meta(), query, first, replace=False) is True
+ assert len(coordinator.keys) == 1
+
+ # A forced refresh must still replace the value.
+ assert repository.store(build_view_meta(), query, first, replace=True) is True
+ assert len(coordinator.keys) == 2
+
+
+def test_store_does_not_skip_when_descriptor_was_evicted() -> None:
+ """A surviving value whose descriptor is gone is undiscoverable, so the
+ store must register it again rather than trust the value's presence."""
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+ bucket_key: str = next(
+ key for key, value in backend.values.items() if isinstance(value, list)
+ )
+ backend.values[bucket_key] = []
+
+ assert (
+ repository.store(
+ build_view_meta(), query, build_semantic_result(), replace=False
+ )
+ is True
+ )
+ assert len(coordinator.keys) == 2
+ assert len(cast(list[CachedEntry], backend.values[bucket_key])) == 1
+
+
+def test_entries_from_an_older_pickled_shape_are_treated_as_empty() -> None:
+ """A pickled dataclass restores only the attributes it was written with.
+ Key rotation keeps such buckets out of reach; if one is reached anyway
+ it must cost a miss, not a lookup failure until it expires."""
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+ bucket_key: str = next(
+ key for key, value in backend.values.items() if isinstance(value, list)
+ )
+ stale: CachedEntry = cast(list[CachedEntry], backend.values[bucket_key])[0]
+ del stale.__dict__["timeout"]
+
+ lookup_result: SemanticCacheLookupResult = repository.lookup(
+ build_view_meta(),
+ query,
+ ContainmentCapabilities(),
+ )
+
+ assert lookup_result.candidates == ()
+ assert lookup_result.missing_value_keys == frozenset()
+
+
+def test_cached_entry_shape_is_pinned_to_the_identity_version() -> None:
+ """``CachedEntry`` is pickled into the shared data cache and outlives the
+ code that wrote it. Changing its fields without rotating every key would
+ hand the new code buckets it cannot read. If this test fails because the
+ shape changed, bump ``IDENTITY_FORMAT_VERSION`` and update both values."""
+ assert tuple(field.name for field in fields(CachedEntry)) == (
+ "filters",
+ "dimensions",
+ "metrics",
+ "limit",
+ "offset",
+ "order_key",
+ "group_limit_key",
+ "value_key",
+ "timestamp",
+ "timeout",
+ "write_nonce",
+ )
+ assert IDENTITY_FORMAT_VERSION == "v4"
+
+
+def test_store_surfaces_existence_check_failure_as_store_error() -> None:
+ """The redundant-store check is an expected backend operation like any
+ other: its failure is a typed store error, not a request failure."""
+
+ class _FailingHasBackend(_Backend):
+ def has(self, key: str) -> bool:
+ raise SemanticCacheBackendError("has unavailable")
+
+ backend: _FailingHasBackend = _FailingHasBackend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _Coordinator(),
+ clock=lambda: 10.0,
+ )
+ query: SemanticQuery = build_semantic_query()
+ repository.store(build_view_meta(), query, build_semantic_result())
+
+ with pytest.raises(SemanticCacheStoreError, match="has failed"):
+ repository.store(
+ build_view_meta(), query, build_semantic_result(), replace=False
+ )
+
+
+def test_lookup_bounds_value_age_by_the_requesting_timeout() -> None:
+ """Requests resolve their own timeout: a value stored by a 3600 s chart
+ must not answer a 60 s chart on the same view once it is older than 60 s,
+ and the 60 s chart's own store must not be skipped as already registered."""
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ clock: list[float] = [0.0]
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ clock=lambda: clock[0],
+ )
+ query: SemanticQuery = build_semantic_query()
+ long_lived: ViewMeta = replace(build_view_meta(), timeout=3600)
+ short_lived: ViewMeta = replace(build_view_meta(), timeout=60)
+ repository.store(long_lived, query, build_semantic_result())
+
+ clock[0] = 30.0
+ assert (
+ len(repository.lookup(short_lived, query, ContainmentCapabilities()).candidates)
+ == 1
+ )
+
+ clock[0] = 1800.0
+ stale: SemanticCacheLookupResult = repository.lookup(
+ short_lived, query, ContainmentCapabilities()
+ )
+ assert stale.candidates == ()
+ # Not a missing value: it is still valid for the long-lived chart.
+ assert stale.missing_value_keys == frozenset()
+ assert (
+ len(repository.lookup(long_lived, query, ContainmentCapabilities()).candidates)
+ == 1
+ )
+ unbounded: ViewMeta = replace(build_view_meta(), timeout=None)
+ assert (
+ len(repository.lookup(unbounded, query, ContainmentCapabilities()).candidates)
+ == 1
+ )
+
+ assert repository.store(short_lived, query, build_semantic_result(), replace=False)
+ assert len(coordinator.keys) == 2
+
+
+def test_store_dedupe_requires_the_registered_ttl_to_cover_this_request() -> None:
+ """A 60 s chart storing first must not pin a 3600 s chart to its entry."""
+ backend: _Backend = _Backend()
+ coordinator: _Coordinator = _Coordinator()
+ clock: list[float] = [0.0]
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ coordinator,
+ clock=lambda: clock[0],
+ )
+ query: SemanticQuery = build_semantic_query()
+ short_lived: ViewMeta = replace(build_view_meta(), timeout=60)
+ long_lived: ViewMeta = replace(build_view_meta(), timeout=3600)
+ repository.store(short_lived, query, build_semantic_result())
+
+ clock[0] = 30.0
+ assert repository.store(long_lived, query, build_semantic_result(), replace=False)
+ assert len(coordinator.keys) == 2
+ # Re-registered with the longer TTL: now the 3600 s chart is covered.
+ assert repository.store(long_lived, query, build_semantic_result(), replace=False)
+ assert len(coordinator.keys) == 2
+
+ never_expiring: ViewMeta = replace(build_view_meta(), timeout=0)
+ repository.store(never_expiring, replace(query, limit=7), build_semantic_result())
+ assert repository.store(
+ long_lived, replace(query, limit=7), build_semantic_result(), replace=False
+ )
+ assert len(coordinator.keys) == 3
+
+
+@pytest.mark.parametrize("contained", [False, True])
+def test_split_store_stale_overwrite_is_a_miss_and_pruned(contained: bool) -> None:
+ """A pause after the lease check cannot turn an old payload into a hit."""
+ leases: _FencedRedis = _FencedRedis()
+ coordination: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ coordination._cache = leases
+ owners: Iterator[str] = iter(["old", "new", "prune"])
+ coordinator: SemanticCacheCoordinator = SemanticCacheCoordinator(
+ coordination,
+ SemanticCacheCoordinationSettings(0, 10),
+ token_factory=lambda: next(owners),
+ )
+
+ class PausingBackend(_Backend):
+ before_value_set: Callable[[], None] | None = None
+
+ def set(self, key: str, value: object, timeout: int | None = None) -> bool:
+ if key.startswith("semantic-cache:value:") and self.before_value_set:
+ callback: Callable[[], None] = self.before_value_set
+ self.before_value_set = None
+ callback()
+ return super().set(key, value, timeout)
+
+ backend: PausingBackend = PausingBackend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(backend, coordinator)
+ meta: ViewMeta = build_view_meta()
+ query: SemanticQuery = build_semantic_query()
+ requested: SemanticQuery = replace(query, limit=1) if contained else query
+ fresh: SemanticResult = replace(build_semantic_result(), requests=["fresh"])
+
+ def takeover() -> None:
+ lease_key: str = next(iter(leases.values))
+ leases.delete(lease_key) # Whole-process stall lets A's lease expire.
+ assert repository.store(meta, query, fresh)
+ assert repository.lookup(meta, requested, ContainmentCapabilities()).candidates
+
+ backend.before_value_set = takeover
+ with pytest.raises(SemanticCacheStoreError): # Old owner cannot release B's lease.
+ repository.store(meta, query, build_semantic_result())
+ lookup: SemanticCacheLookupResult = repository.lookup(
+ meta, requested, ContainmentCapabilities()
+ )
+ assert lookup.candidates == ()
+ assert len(lookup.missing_value_keys) == 1
+ repository.prune_missing(meta, lookup.missing_value_keys)
+ assert backend.values[repository._bucket_key(meta)] == []
+ assert all(key in backend.values for key in lookup.missing_value_keys)
+
+
+@pytest.mark.parametrize(
+ "damage", ["bare", "mismatch", "missing_nonce", "empty_descriptor", "bad_result"]
+)
+def test_fence_rejects_invalid_pairs_and_store_repairs(damage: str) -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend, _Coordinator()
+ )
+ meta: ViewMeta = build_view_meta()
+ query: SemanticQuery = build_semantic_query()
+ repository.store(meta, query, build_semantic_result())
+ bucket: str = repository._bucket_key(meta)
+ entry: CachedEntry = cast(list[CachedEntry], backend.values[bucket])[0]
+ value: CachedValue = cast(CachedValue, backend.values[entry.value_key])
+ if damage == "bare":
+ backend.values[entry.value_key] = value.result
+ elif damage == "mismatch":
+ backend.values[entry.value_key] = replace(value, write_nonce="other")
+ elif damage == "missing_nonce":
+ del value.__dict__["write_nonce"]
+ elif damage == "empty_descriptor":
+ backend.values[bucket] = [replace(entry, write_nonce="")]
+ else:
+ value.__dict__["result"] = None
+ lookup: SemanticCacheLookupResult = repository.lookup(
+ meta, query, ContainmentCapabilities()
+ )
+ assert lookup.candidates == ()
+ assert lookup.fence_rejections == 1
+ assert lookup.missing_value_keys == frozenset({entry.value_key})
+ assert repository.store(meta, query, build_semantic_result(), replace=False)
+ repaired: CachedEntry = cast(list[CachedEntry], backend.values[bucket])[0]
+ assert repaired.write_nonce != entry.write_nonce
+ repository.prune_missing(meta, lookup.missing_value_keys)
+ assert repository.lookup(meta, query, ContainmentCapabilities()).candidates
+
+
+def test_legacy_identity_namespace_cannot_supply_a_hit() -> None:
+ backend: _Backend = _Backend()
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend, _Coordinator()
+ )
+ meta: ViewMeta = build_view_meta()
+ query: SemanticQuery = build_semantic_query()
+ with patch("superset.semantic_layers.cache_identity.IDENTITY_FORMAT_VERSION", "v3"):
+ repository.store(meta, query, build_semantic_result())
+ assert repository.lookup(meta, query, ContainmentCapabilities()).candidates == ()
diff --git a/tests/unit_tests/semantic_layers/cache_test.py b/tests/unit_tests/semantic_layers/cache_test.py
new file mode 100644
index 00000000000..811a764cd96
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/cache_test.py
@@ -0,0 +1,956 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections.abc import Callable
+from time import monotonic
+from typing import Any, cast
+from unittest.mock import MagicMock
+
+import pyarrow as pa
+import pytest
+from flask import Flask
+from flask_caching.backends.nullcache import NullCache
+from flask_caching.backends.rediscache import RedisCache, RedisSentinelCache
+from pytest_mock import MockerFixture
+from redis import Redis
+from redis.exceptions import TimeoutError as RedisTimeoutError
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Dimension,
+ Metric,
+ SemanticQuery,
+ SemanticResult,
+)
+
+from superset.semantic_layers.cache import (
+ eligible_reuse_mode,
+ initialize_semantic_cache,
+ rank_eligible_entries,
+ SafeSemanticCacheBackend,
+ SemanticCacheDisabledReason,
+ SemanticCacheOutcome,
+ SemanticCacheService,
+ SemanticCacheState,
+)
+from superset.semantic_layers.cache_coordination import OwnerTokenCoordinationBackend
+from superset.semantic_layers.cache_policy import (
+ ContainmentCapabilities,
+ ReuseDecision,
+ ReuseMode,
+)
+from superset.semantic_layers.cache_repository import (
+ CachedEntry,
+ CachedResultCandidate,
+ SemanticCacheBackend,
+ SemanticCacheBackendError,
+ SemanticCacheLookupError,
+ SemanticCacheLookupResult,
+ SemanticCacheRepository,
+ SemanticCacheWriteFence,
+)
+from tests.unit_tests.semantic_layers.conftest import (
+ build_semantic_query,
+ build_semantic_result,
+ build_view_meta,
+)
+
+
+@pytest.mark.parametrize("phase", ["acquire", "refresh", "release"])
+def test_coordination_timeout_keeps_provider_result(phase: str) -> None:
+ """Bounded Redis I/O errors stay inside the best-effort store boundary."""
+ import superset.semantic_layers.cache as cache_module
+
+ coordination: MagicMock = MagicMock()
+ coordination.acquire_owner_token.return_value = True
+ coordination.refresh_owner_token.return_value = True
+ coordination.release_owner_token.return_value = True
+ getattr(coordination, f"{phase}_owner_token").side_effect = RedisTimeoutError()
+ metrics: MagicMock = MagicMock()
+ backend: MagicMock = MagicMock(spec=SemanticCacheBackend)
+ backend.get.return_value = None
+ backend.set.return_value = True
+ initialize_semantic_cache(
+ parent_enabled=True,
+ requested=True,
+ backend=backend,
+ coordination=coordination,
+ wait_seconds=0.01,
+ lease_seconds=1,
+ metrics=metrics,
+ )
+ result: SemanticResult = build_semantic_result()
+ start: float = monotonic()
+ outcome: SemanticCacheOutcome = cache_module.semantic_cache_service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ lambda _: result,
+ capabilities=ContainmentCapabilities(),
+ force=True,
+ )
+ assert monotonic() - start < 1
+ assert outcome.result is result
+ assert not outcome.cache_hit
+ metrics.incr.assert_any_call("semantic_cache.containment.coordination_failure")
+ metrics.incr.assert_any_call("semantic_cache.containment.store_failure")
+
+
+@pytest.mark.parametrize("wait_seconds,expected", [(0.0, 0.001), (0.25, 0.25)])
+def test_coordination_constructs_private_bounded_client(
+ mocker: MockerFixture,
+ wait_seconds: float,
+ expected: float,
+) -> None:
+ """Do not mutate the unbounded shared client's connection settings."""
+ from superset.coordination.cache_backend import RedisCacheBackend
+ from superset.semantic_layers.cache import _bounded_coordination_backend
+
+ app: Flask = Flask(__name__)
+ app.config["DISTRIBUTED_COORDINATION_CONFIG"] = {"CACHE_TYPE": "RedisCache"}
+ original: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ original._cache = MagicMock()
+ redis_constructor: MagicMock = mocker.patch(
+ "superset.coordination.cache_backend.redis.Redis"
+ )
+ with app.app_context():
+ bounded: OwnerTokenCoordinationBackend = _bounded_coordination_backend(
+ original, wait_seconds
+ )
+ assert bounded is not original
+ kwargs: dict[str, Any] = redis_constructor.call_args.kwargs
+ assert kwargs["socket_timeout"] == expected
+ assert kwargs["socket_connect_timeout"] == expected
+ original._cache.assert_not_called()
+
+
+def test_fenced_write_wraps_backend_failure(mocker: MockerFixture) -> None:
+ """A failed atomic write exposes the cache boundary error and its cause."""
+ from superset.coordination.cache_backend import RedisCacheBackend
+
+ coordination: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ client: MagicMock = MagicMock()
+ coordination._cache = client
+ backend: SafeSemanticCacheBackend = SafeSemanticCacheBackend(
+ RedisCache(host=client)
+ )
+ failure: RedisTimeoutError = RedisTimeoutError("write timed out")
+ mocker.patch.object(coordination, "compare_owner_and_set", side_effect=failure)
+ fence: SemanticCacheWriteFence = SemanticCacheWriteFence(
+ "lease", "owner", lambda: True, coordination
+ )
+
+ raised: pytest.ExceptionInfo[SemanticCacheBackendError]
+ with pytest.raises(SemanticCacheBackendError, match="fenced set failed") as raised:
+ backend.set_if_owner(fence, "value", "payload", 60)
+
+ assert raised.value.__cause__ is failure
+ client.set.assert_not_called()
+
+
+def test_coordination_without_app_preserves_injected_client(
+ mocker: MockerFixture,
+) -> None:
+ """Without application configuration the injected client stays caller-owned."""
+ from superset.coordination.cache_backend import RedisCacheBackend
+ from superset.semantic_layers.cache import _bounded_coordination_backend
+
+ original: RedisCacheBackend = object.__new__(RedisCacheBackend)
+ mocker.patch("superset.semantic_layers.cache.has_app_context", return_value=False)
+ factory: MagicMock = mocker.patch.object(RedisCacheBackend, "from_config")
+
+ assert _bounded_coordination_backend(original, 0.25) is original
+ factory.assert_not_called()
+
+
+def test_coordination_bounds_sentinel_discovery_and_master() -> None:
+ """Private master and Sentinel discovery clients both have bounded retries."""
+ from superset.coordination.cache_backend import RedisSentinelCacheBackend
+ from superset.semantic_layers.cache import _bounded_coordination_backend
+
+ app: Flask = Flask(__name__)
+ app.config["DISTRIBUTED_COORDINATION_CONFIG"] = {
+ "CACHE_TYPE": "RedisSentinelCache",
+ "CACHE_REDIS_SENTINELS": [("127.0.0.1", 26379), ("127.0.0.1", 26380)],
+ }
+ original: RedisSentinelCacheBackend = object.__new__(RedisSentinelCacheBackend)
+ with app.app_context():
+ bounded: OwnerTokenCoordinationBackend = _bounded_coordination_backend(
+ original, 0.25
+ )
+
+ assert isinstance(bounded, RedisSentinelCacheBackend)
+ assert bounded is not original
+ assert len(bounded._sentinel.sentinels) == 2
+ client: Redis
+ for client in [bounded._cache, *bounded._sentinel.sentinels]:
+ options: dict[str, Any] = client.connection_pool.connection_kwargs
+ assert options["socket_timeout"] == 0.25
+ assert options["socket_connect_timeout"] == 0.25
+ assert options["retry"].get_retries() == 0
+
+
+class _ImmediateCoordinator:
+ def mutate(self, key: str, operation: Callable[[], None]) -> bool:
+ operation()
+ return True
+
+
+class _OwnerTokenBackend:
+ def acquire_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ return True
+
+ def release_owner_token(self, key: str, owner_token: str) -> bool:
+ return True
+
+ def refresh_owner_token(
+ self,
+ key: str,
+ owner_token: str,
+ lease_seconds: int,
+ ) -> bool:
+ return True
+
+
+def _metric(
+ metric_id: str,
+ aggregation: AggregationType | None = AggregationType.SUM,
+) -> Metric:
+ return Metric(
+ id=metric_id,
+ name=metric_id,
+ type=pa.float64(),
+ definition=f"{aggregation or 'unknown'}({metric_id})",
+ aggregation=aggregation,
+ )
+
+
+def _dimension(dimension_id: str) -> Dimension:
+ return Dimension(id=dimension_id, name=dimension_id, type=pa.string())
+
+
+def _query(
+ *,
+ dimensions: tuple[str, ...] = ("country",),
+ metrics: tuple[Metric, ...] = (_metric("revenue"),),
+ limit: int | None = None,
+ offset: int | None = None,
+) -> SemanticQuery:
+ return SemanticQuery(
+ metrics=list(metrics),
+ dimensions=[_dimension(key) for key in dimensions],
+ limit=limit,
+ offset=offset,
+ )
+
+
+def _entry(
+ *,
+ dimensions: tuple[str, ...] = ("country",),
+ metrics: tuple[Metric, ...] = (_metric("revenue"),),
+ limit: int | None = None,
+ offset: int = 0,
+ order_key: str = "",
+ group_limit_key: str = "",
+ value_key: str = "value",
+ timestamp: float = 1.0,
+) -> CachedEntry:
+ return CachedEntry(
+ filters=frozenset(),
+ dimensions=frozenset(_dimension(key) for key in dimensions),
+ metrics=frozenset(metrics),
+ limit=limit,
+ offset=offset,
+ order_key=order_key,
+ group_limit_key=group_limit_key,
+ value_key=value_key,
+ timestamp=timestamp,
+ )
+
+
+def test_disabled_state_is_safely_ineffective() -> None:
+ state: SemanticCacheState = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.FLAG_OFF
+ )
+
+ assert state.requested is False
+ assert state.effective is False
+ assert state.disabled_reason is SemanticCacheDisabledReason.FLAG_OFF
+
+
+def test_requested_unsupported_state_preserves_request() -> None:
+ state: SemanticCacheState = SemanticCacheState.disabled(
+ SemanticCacheDisabledReason.UNSUPPORTED_COORDINATION,
+ requested=True,
+ )
+
+ assert state.requested is True
+ assert state.effective is False
+
+
+@pytest.mark.parametrize(
+ "requested,effective,disabled_reason,error_match",
+ [
+ (False, True, None, "must be requested"),
+ (
+ True,
+ True,
+ SemanticCacheDisabledReason.FLAG_OFF,
+ "cannot have a disabled reason",
+ ),
+ (True, False, None, "requires a disabled reason"),
+ ],
+)
+def test_cache_state_rejects_contradictory_combinations(
+ requested: bool,
+ effective: bool,
+ disabled_reason: SemanticCacheDisabledReason | None,
+ error_match: str,
+) -> None:
+ with pytest.raises(ValueError, match=error_match):
+ SemanticCacheState(
+ requested=requested,
+ effective=effective,
+ disabled_reason=disabled_reason,
+ )
+
+
+def test_enabled_state_has_one_valid_representation() -> None:
+ state: SemanticCacheState = SemanticCacheState.enabled()
+
+ assert state.requested is True
+ assert state.effective is True
+ assert state.disabled_reason is None
+
+
+def test_default_service_executes_provider_without_touching_cache() -> None:
+ service: SemanticCacheService = SemanticCacheService.default_ineffective()
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ provider,
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert outcome.result == build_semantic_result()
+ assert outcome.cache_hit is False
+ provider.assert_called_once()
+
+
+def test_application_metrics_use_fixed_names_and_are_non_fatal() -> None:
+ metrics: MagicMock = MagicMock()
+ metrics.incr.side_effect = RuntimeError("metrics unavailable")
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.disabled(SemanticCacheDisabledReason.FLAG_OFF),
+ metrics=metrics,
+ )
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ provider,
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert outcome.cache_hit is False
+ metrics.incr.assert_called_once_with("semantic_cache.containment.bypass")
+
+
+def test_safe_backend_has_delegates_and_normalizes_to_bool() -> None:
+ delegate: MagicMock = MagicMock()
+ delegate.has.return_value = 1
+ backend: SafeSemanticCacheBackend = SafeSemanticCacheBackend(delegate)
+
+ assert backend.has("key") is True
+ delegate.has.assert_called_once_with("key")
+
+
+@pytest.mark.parametrize("operation", ["get", "has", "set", "delete"])
+def test_safe_backend_translates_pluggable_cache_failures(operation: str) -> None:
+ delegate: MagicMock = MagicMock()
+ getattr(delegate, operation).side_effect = RuntimeError("client unavailable")
+ backend: SafeSemanticCacheBackend = SafeSemanticCacheBackend(delegate)
+ operation_call: Callable[[], object] = {
+ "get": lambda: backend.get("key"),
+ "has": lambda: backend.has("key"),
+ "set": lambda: backend.set("key", "value", timeout=30),
+ "delete": lambda: backend.delete("key"),
+ }[operation]
+
+ with pytest.raises(SemanticCacheBackendError, match=operation):
+ operation_call()
+
+
+@pytest.mark.parametrize(
+ "parent_enabled,requested,coordination,expected_reason",
+ [
+ (
+ False,
+ True,
+ _OwnerTokenBackend(),
+ SemanticCacheDisabledReason.PARENT_FLAG_OFF,
+ ),
+ (True, False, _OwnerTokenBackend(), SemanticCacheDisabledReason.FLAG_OFF),
+ (True, True, None, SemanticCacheDisabledReason.UNSUPPORTED_COORDINATION),
+ ],
+)
+def test_initialization_preserves_requested_and_effective_state(
+ parent_enabled: bool,
+ requested: bool,
+ coordination: OwnerTokenCoordinationBackend | None,
+ expected_reason: SemanticCacheDisabledReason,
+) -> None:
+ backend: MagicMock = MagicMock()
+
+ state: SemanticCacheState = initialize_semantic_cache(
+ parent_enabled=parent_enabled,
+ requested=requested,
+ backend=backend,
+ coordination=coordination,
+ wait_seconds=1.0,
+ lease_seconds=30,
+ )
+
+ assert state.requested is requested
+ assert state.effective is False
+ assert state.disabled_reason is expected_reason
+
+
+@pytest.mark.parametrize(
+ "wait_seconds,lease_seconds",
+ [
+ (float("nan"), 30),
+ (1.0, 0),
+ (True, 30),
+ (1.0, True),
+ ("1", 30),
+ (1.0, "30"),
+ (None, 30),
+ ],
+)
+def test_initialization_rejects_invalid_coordination_settings(
+ wait_seconds: object,
+ lease_seconds: object,
+) -> None:
+ state: SemanticCacheState = initialize_semantic_cache(
+ parent_enabled=True,
+ requested=True,
+ backend=MagicMock(),
+ coordination=_OwnerTokenBackend(),
+ wait_seconds=wait_seconds, # type: ignore[arg-type]
+ lease_seconds=lease_seconds, # type: ignore[arg-type]
+ )
+
+ assert state.disabled_reason is (
+ SemanticCacheDisabledReason.INVALID_COORDINATION_CONFIGURATION
+ )
+
+
+def test_restart_configuration_converges_from_enabled_to_disabled() -> None:
+ enabled: SemanticCacheState = initialize_semantic_cache(
+ parent_enabled=True,
+ requested=True,
+ backend=MagicMock(),
+ coordination=_OwnerTokenBackend(),
+ wait_seconds=1.0,
+ lease_seconds=30,
+ )
+ disabled: SemanticCacheState = initialize_semantic_cache(
+ parent_enabled=True,
+ requested=False,
+ backend=MagicMock(),
+ coordination=_OwnerTokenBackend(),
+ wait_seconds=1.0,
+ lease_seconds=30,
+ )
+
+ assert enabled.effective is True
+ assert disabled.effective is False
+
+
+def test_effective_service_requires_repository() -> None:
+ with pytest.raises(ValueError, match="requires a repository"):
+ SemanticCacheService(SemanticCacheState.enabled())
+
+
+def test_prune_failure_does_not_prevent_provider_execution() -> None:
+ repository: MagicMock = MagicMock(spec=SemanticCacheRepository)
+ repository.lookup.return_value = SemanticCacheLookupResult(
+ candidates=(),
+ missing_value_keys=frozenset({"missing"}),
+ )
+ repository.prune_missing.side_effect = SemanticCacheLookupError("prune unavailable")
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ )
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ provider,
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert outcome.cache_hit is False
+ provider.assert_called_once()
+
+
+def test_force_bypasses_lookup_and_stores_provider_result() -> None:
+ repository: MagicMock = MagicMock(spec=SemanticCacheRepository)
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ )
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ provider,
+ capabilities=ContainmentCapabilities(),
+ force=True,
+ )
+
+ assert outcome.cache_hit is False
+ repository.lookup.assert_not_called()
+ repository.store.assert_called_once()
+ assert repository.store.call_args.kwargs["replace"] is True
+
+
+def test_ordinary_miss_does_not_replace_an_identical_concurrent_store() -> None:
+ repository: MagicMock = MagicMock(spec=SemanticCacheRepository)
+ repository.lookup.return_value = SemanticCacheLookupResult(
+ candidates=(), missing_value_keys=frozenset()
+ )
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ )
+
+ service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ MagicMock(return_value=build_semantic_result()),
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert repository.store.call_args.kwargs["replace"] is False
+
+
+def test_oversized_result_is_served_but_never_stored() -> None:
+ """Values are pickled inside the lease and share the data cache; a result
+ above the operator's cap is returned to the user and skipped."""
+ repository: MagicMock = MagicMock(spec=SemanticCacheRepository)
+ repository.lookup.return_value = SemanticCacheLookupResult(
+ candidates=(), missing_value_keys=frozenset()
+ )
+ metrics: MagicMock = MagicMock()
+ result: SemanticResult = build_semantic_result()
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ metrics,
+ max_value_bytes=result.results.nbytes - 1,
+ )
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ MagicMock(return_value=result),
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert outcome.result is result
+ assert outcome.cache_hit is False
+ repository.store.assert_not_called()
+ metrics.incr.assert_any_call("semantic_cache.containment.store_skipped")
+
+ within_cap: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ metrics,
+ max_value_bytes=result.results.nbytes,
+ )
+ within_cap.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ MagicMock(return_value=result),
+ capabilities=ContainmentCapabilities(),
+ )
+ repository.store.assert_called_once()
+
+
+def test_lookup_backend_failure_executes_provider() -> None:
+ backend: MagicMock = MagicMock()
+ backend.get.side_effect = SemanticCacheBackendError("lookup unavailable")
+ repository: SemanticCacheRepository = SemanticCacheRepository(
+ backend,
+ _ImmediateCoordinator(),
+ )
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ )
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
+
+ outcome: SemanticCacheOutcome = service.execute(
+ build_view_meta(),
+ build_semantic_query(),
+ provider,
+ capabilities=ContainmentCapabilities(),
+ )
+
+ assert outcome.result == build_semantic_result()
+ assert outcome.cache_hit is False
+ provider.assert_called_once()
+
+
+def test_transform_failure_falls_back_to_provider() -> None:
+ query: SemanticQuery = build_semantic_query()
+ incompatible_query: SemanticQuery = SemanticQuery(
+ metrics=query.metrics,
+ dimensions=[Dimension("missing", "Missing", pa.string())],
+ )
+ entry: CachedEntry = _entry()
+ repository: MagicMock = MagicMock(spec=SemanticCacheRepository)
+ repository.lookup.return_value = SemanticCacheLookupResult(
+ candidates=(
+ CachedResultCandidate(
+ entry,
+ build_semantic_result(),
+ ReuseDecision(ReuseMode.EXACT, frozenset()),
+ ),
+ ),
+ missing_value_keys=frozenset(),
+ )
+ metrics: MagicMock = MagicMock()
+ service: SemanticCacheService = SemanticCacheService(
+ SemanticCacheState.enabled(),
+ repository,
+ metrics,
+ )
+ provider: MagicMock = MagicMock(return_value=build_semantic_result())
... 888 lines suppressed ...