You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by jl...@apache.org on 2026/08/06 00:45:08 UTC
(superset) 02/10: fix(reports): merge saved formats, match d3 exponents, resolve locale
This is an automated email from the ASF dual-hosted git repository.
sadpandajoe pushed a commit to branch fix-reports-text-number-formatting
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 7af7d1bd4068d8d5ea1880fb5a36fa26438d8f8c
Author: sadpandajoe <jc...@gmail.com>
AuthorDate: Thu Aug 6 00:09:45 2026 +0000
fix(reports): merge saved formats, match d3 exponents, resolve locale
Addresses review feedback left unresolved on #41028.
Saved metric formats were dropped: both frontend plugins read per-metric
formats from datasource column_formats, while the server read columnFormats
from form data only, and table() ignored datasource entirely. The datasource
formats are now merged with chart-level overrides, with the chart override
winning, matching plugin-chart-table transformProps and PivotTableChart.
Default formatting now follows d3's exponent thresholds instead of forcing
fixed-point, so very small and very large values render as they do in the
browser.
Currency symbol placement is resolved from the request or configured locale
rather than a hard-coded en_US, so reports agree with Explore on non-English
deployments. Reports execute in a Celery worker with no request context, so
resolution falls back through BABEL_DEFAULT_LOCALE to a safe default.
---
superset/charts/client_processing.py | 71 +++++++++++++++++++---
superset/utils/number_format.py | 41 ++++++++++---
tests/unit_tests/charts/test_client_processing.py | 74 +++++++++++++++++++++++
tests/unit_tests/utils/number_format_test.py | 46 +++++++++++++-
4 files changed, 211 insertions(+), 21 deletions(-)
diff --git a/superset/charts/client_processing.py b/superset/charts/client_processing.py
index 9e1945cba34..5f666da8bf7 100644
--- a/superset/charts/client_processing.py
+++ b/superset/charts/client_processing.py
@@ -268,7 +268,15 @@ def format_column(
currency: dict[str, Any],
detected_currency: Optional[str] = None,
) -> None:
- """Format a column in place when a number or currency format is configured."""
+ """
+ Format a column in place when a number or currency format is configured.
+
+ ``detected_currency`` represents the query-wide single currency. Per-cell
+ AUTO currency needs row context, while this helper receives one value
+ series and the pivot path does not retain each cell's contributing currency
+ rows. Mixed-currency results therefore use the formatted number without a
+ currency symbol.
+ """
currency = resolve_auto_currency(currency, detected_currency)
if d3_format or currency.get("symbol"):
df[column] = df[column].apply(
@@ -276,6 +284,40 @@ def format_column(
)
+def get_datasource_column_formats(
+ datasource: Optional[Union["BaseDatasource", "Query"]],
+) -> tuple[dict[str, str | None], dict[str, str]]:
+ """Return saved metric formats and verbose labels from a datasource."""
+ if not datasource:
+ return {}, {}
+
+ datasource_data = datasource.data
+ return (
+ datasource_data.get("column_formats") or {},
+ datasource_data.get("verbose_map") or {},
+ )
+
+
+def merge_column_formats(
+ form_data: dict[str, Any],
+ datasource: Optional[Union["BaseDatasource", "Query"]],
+) -> dict[str, str | None]:
+ """Merge saved formats with truthy chart overrides using verbose labels."""
+ saved_formats, verbose_map = get_datasource_column_formats(datasource)
+ column_formats = {
+ verbose_map.get(metric, metric): d3_format
+ for metric, d3_format in saved_formats.items()
+ }
+ column_formats.update(
+ {
+ verbose_map.get(metric, metric): d3_format
+ for metric, d3_format in (form_data.get("columnFormats") or {}).items()
+ if d3_format
+ }
+ )
+ return column_formats
+
+
def pivot_table_v2(
df: pd.DataFrame,
form_data: dict[str, Any],
@@ -301,7 +343,9 @@ def pivot_table_v2(
apply_metrics_on_rows=form_data.get("metricsLayout") == "ROWS",
)
if apply_number_format:
- return apply_pivot_number_formats(pivoted, form_data, detected_currency)
+ return apply_pivot_number_formats(
+ pivoted, form_data, detected_currency, datasource
+ )
return pivoted
@@ -309,6 +353,7 @@ def apply_pivot_number_formats(
df: pd.DataFrame,
form_data: dict[str, Any],
detected_currency: Optional[str] = None,
+ datasource: Optional[Union["BaseDatasource", "Query"]] = None,
) -> pd.DataFrame:
"""
Apply `valueFormat`/`columnFormats` and currency config to pivot values.
@@ -318,7 +363,7 @@ def apply_pivot_number_formats(
Per-metric overrides fall back to the global value format.
"""
value_format = form_data.get("valueFormat")
- column_formats = form_data.get("columnFormats") or {}
+ column_formats = merge_column_formats(form_data, datasource)
currency_format = form_data.get("currencyFormat") or {}
currency_formats = form_data.get("currencyFormats") or {}
metric_level = -1 if form_data.get("combineMetric") else 0
@@ -332,7 +377,7 @@ def apply_pivot_number_formats(
format_column(
df,
column,
- column_formats.get(metric, value_format),
+ column_formats.get(metric) or value_format,
currency_formats.get(metric) or currency_format,
detected_currency,
)
@@ -343,9 +388,7 @@ def apply_pivot_number_formats(
def table(
df: pd.DataFrame,
form_data: dict[str, Any],
- datasource: Optional[ # pylint: disable=unused-argument
- Union["BaseDatasource", "Query"]
- ] = None,
+ datasource: Optional[Union["BaseDatasource", "Query"]] = None,
apply_number_format: bool = True,
detected_currency: Optional[str] = None,
) -> pd.DataFrame:
@@ -355,13 +398,21 @@ def table(
if not apply_number_format:
return df
- column_config = form_data.get("column_config", {})
- for column, config in column_config.items():
+ saved_formats, verbose_map = get_datasource_column_formats(datasource)
+ column_config = form_data.get("column_config") or {}
+ columns = dict.fromkeys([*saved_formats, *column_config])
+ for original_column in columns:
+ column = (
+ original_column
+ if original_column in df.columns
+ else verbose_map.get(original_column, original_column)
+ )
if column in df.columns:
+ config = column_config.get(original_column) or {}
format_column(
df,
column,
- config.get("d3NumberFormat"),
+ config.get("d3NumberFormat") or saved_formats.get(original_column),
config.get("currencyFormat") or {},
detected_currency,
)
diff --git a/superset/utils/number_format.py b/superset/utils/number_format.py
index 93eef6c0bab..2eb118f4b85 100644
--- a/superset/utils/number_format.py
+++ b/superset/utils/number_format.py
@@ -42,12 +42,15 @@ from functools import lru_cache
from typing import Any
from babel.numbers import format_currency, get_currency_symbol
+from flask import current_app
+from flask_babel import get_locale
SMART_NUMBER = "SMART_NUMBER"
SMART_NUMBER_SIGNED = "SMART_NUMBER_SIGNED"
AUTO_CURRENCY = "AUTO"
-LOCALE = "en_US"
+DEFAULT_LOCALE = "en"
+CURRENCY_SYMBOL_LOCALE = "en_US"
# SI prefixes keyed by their power-of-1000 exponent, mirroring d3-format.
SI_PREFIXES = {
@@ -223,10 +226,12 @@ def format_default(value: float, comma: str) -> str:
Format ``value`` the way d3's default (no-type) specifier does.
Mirrors JavaScript's ``Number.toString``: the shortest decimal representation
- with optional grouping and no trailing zeros, so whole-valued floats render
- without a spurious ``.0`` (``4725.0`` -> ``4,725``) and small values stay in
- fixed-point notation (``0.00005`` rather than ``5e-5``).
+ with optional grouping and no trailing zeros. JavaScript switches to exponent
+ notation below ``1e-6`` and at or above ``1e21``; values at ``1e-6`` remain
+ fixed-point.
"""
+ if value and (value < 1e-6 or value >= 1e21):
+ return normalize_exponent(repr(value))
formatted = format(Decimal(repr(value)), f"{comma}f")
return trim_trailing_zeros(formatted)
@@ -335,14 +340,34 @@ def normalize_exponent(formatted: str) -> str:
return re.sub(r"([eE][+-])0*(\d)", r"\1\2", formatted)
+def get_currency_locale() -> str:
+ """
+ Return the request locale, or the configured default outside a request.
+
+ Report tasks run with a Flask application context but without a request, so
+ Flask-Babel can return ``None``. The config fallback keeps Celery-rendered
+ reports aligned with the locale supplied to the frontend at bootstrap.
+ """
+ try:
+ if locale := get_locale():
+ return str(locale)
+ except RuntimeError:
+ pass
+
+ try:
+ return str(current_app.config.get("BABEL_DEFAULT_LOCALE") or DEFAULT_LOCALE)
+ except RuntimeError:
+ return DEFAULT_LOCALE
+
+
@lru_cache(maxsize=None)
-def resolve_symbol_position(code: str) -> str:
+def resolve_symbol_position(code: str, locale: str) -> str:
"""
Derive the symbol position from the locale's convention for the currency,
mirroring the frontend's ``resolveSymbolPosition``.
"""
try:
- sample = format_currency(1, code, locale=LOCALE)
+ sample = format_currency(1, code, locale=locale)
first_digit = next(i for i, char in enumerate(sample) if char.isdigit())
return "prefix" if first_digit > 0 else "suffix"
except Exception: # pylint: disable=broad-except # noqa: BLE001
@@ -352,10 +377,10 @@ def resolve_symbol_position(code: str) -> str:
def apply_currency(formatted: str, currency: dict[str, Any]) -> str:
normalized = formatted.replace("%", "")
code = currency["symbol"]
- symbol = get_currency_symbol(code, locale=LOCALE) or code
+ symbol = get_currency_symbol(code, locale=CURRENCY_SYMBOL_LOCALE) or code
position = currency.get("symbolPosition")
if position not in ("prefix", "suffix"):
- position = resolve_symbol_position(code)
+ position = resolve_symbol_position(code, get_currency_locale())
if position == "prefix":
return f"{symbol} {normalized}"
return f"{normalized} {symbol}"
diff --git a/tests/unit_tests/charts/test_client_processing.py b/tests/unit_tests/charts/test_client_processing.py
index 9054dc41d6d..75d63ddb21c 100644
--- a/tests/unit_tests/charts/test_client_processing.py
+++ b/tests/unit_tests/charts/test_client_processing.py
@@ -16,6 +16,7 @@
# under the License.
from io import BytesIO, StringIO
+from unittest.mock import MagicMock
import pandas as pd
import pytest
@@ -1886,6 +1887,36 @@ def test_table_applies_si_number_format():
assert formatted["amount"].tolist() == ["1.23k"]
+def test_table_applies_datasource_saved_metric_format_without_chart_override():
+ df = pd.DataFrame.from_dict({"amount": {0: 1234.5}})
+ datasource = MagicMock()
+ datasource.data = {
+ "column_formats": {"amount": ",.2f"},
+ "verbose_map": {},
+ }
+
+ formatted = table(df, {"viz_type": "table"}, datasource)
+
+ assert formatted["amount"].tolist() == ["1,234.50"]
+
+
+def test_table_chart_format_overrides_datasource_saved_metric_format():
+ df = pd.DataFrame.from_dict({"amount": {0: 1234.5}})
+ datasource = MagicMock()
+ datasource.data = {
+ "column_formats": {"amount": ",.2f"},
+ "verbose_map": {},
+ }
+ form_data = {
+ "viz_type": "table",
+ "column_config": {"amount": {"d3NumberFormat": ",.1f"}},
+ }
+
+ formatted = table(df, form_data, datasource)
+
+ assert formatted["amount"].tolist() == ["1,234.5"]
+
+
def test_pivot_table_v2_applies_value_format():
"""
Pivot table reports honor `valueFormat` and per-metric `columnFormats`.
@@ -1908,6 +1939,49 @@ def test_pivot_table_v2_applies_value_format():
assert formatted[("qty",)].tolist() == ["10", "20"]
+def test_pivot_table_v2_applies_datasource_saved_metric_format_without_override():
+ df = pd.DataFrame({"region": ["A", "B"], "sales": [1234.5, 6789.0]})
+ datasource = MagicMock()
+ datasource.data = {
+ "column_formats": {"sales": ",d"},
+ "verbose_map": {},
+ }
+ form_data = {
+ "viz_type": "pivot_table_v2",
+ "groupbyRows": ["region"],
+ "groupbyColumns": [],
+ "metrics": ["sales"],
+ "aggregateFunction": "Sum",
+ "valueFormat": ",.2f",
+ }
+
+ formatted = pivot_table_v2(df, form_data, datasource)
+
+ assert formatted[("sales",)].tolist() == ["1,235", "6,789"]
+
+
+def test_pivot_table_v2_chart_format_overrides_datasource_saved_metric_format():
+ df = pd.DataFrame({"region": ["A"], "sales": [1234.5]})
+ datasource = MagicMock()
+ datasource.data = {
+ "column_formats": {"sales": ",d"},
+ "verbose_map": {},
+ }
+ form_data = {
+ "viz_type": "pivot_table_v2",
+ "groupbyRows": ["region"],
+ "groupbyColumns": [],
+ "metrics": ["sales"],
+ "aggregateFunction": "Sum",
+ "valueFormat": ",.2f",
+ "columnFormats": {"sales": ",.1f"},
+ }
+
+ formatted = pivot_table_v2(df, form_data, datasource)
+
+ assert formatted[("sales",)].tolist() == ["1,234.5"]
+
+
def test_pivot_table_v2_applies_per_metric_format_when_metrics_combined():
"""
Per-metric formats apply when `combineMetric` moves the metric to the last
diff --git a/tests/unit_tests/utils/number_format_test.py b/tests/unit_tests/utils/number_format_test.py
index 091461513fd..3470f37b506 100644
--- a/tests/unit_tests/utils/number_format_test.py
+++ b/tests/unit_tests/utils/number_format_test.py
@@ -15,8 +15,10 @@
# specific language governing permissions and limitations
# under the License.
from decimal import Decimal
+from unittest.mock import patch
import pytest
+from flask import current_app
from superset.utils.number_format import format_number_with_config
@@ -71,6 +73,44 @@ def test_auto_currency_formats_without_symbol():
)
+@pytest.mark.parametrize(
+ "value,expected",
+ [
+ (5e-7, "5e-7"),
+ (9.999999e-7, "9.999999e-7"),
+ (1e-6, "0.000001"),
+ (1e20, "100,000,000,000,000,000,000"),
+ (999_999_999_999_999_900_000, "999,999,999,999,999,900,000"),
+ (1e21, "1e+21"),
+ ],
+)
+def test_default_format_matches_d3_exponent_boundaries(value, expected):
+ assert format_number_with_config(",", None, value) == expected
+
+
+def test_currency_position_uses_request_locale():
+ with patch("superset.utils.number_format.get_locale", return_value="fr_FR"):
+ assert (
+ format_number_with_config(
+ ",.2f", {"symbol": "EUR", "symbolPosition": None}, 1234.5
+ )
+ == "1,234.50 €"
+ )
+
+
+def test_currency_position_uses_configured_locale_without_request():
+ with (
+ patch("superset.utils.number_format.get_locale", return_value=None),
+ patch.dict(current_app.config, {"BABEL_DEFAULT_LOCALE": "fr_FR"}),
+ ):
+ assert (
+ format_number_with_config(
+ ",.2f", {"symbol": "EUR", "symbolPosition": None}, 1234.5
+ )
+ == "1,234.50 €"
+ )
+
+
def test_non_numeric_value_is_returned_as_is():
assert format_number_with_config(",.2f", None, "abc") == "abc"
assert format_number_with_config(",.2f", None, None) == ""
@@ -265,9 +305,9 @@ EXPECTED = {
def test_matches_frontend_d3_format(d3_format):
for value, expected in zip(VALUES, EXPECTED[d3_format], strict=True):
result = format_number_with_config(d3_format, None, value)
- assert result == expected.replace("−", "-"), (
- f"{d3_format!r} of {value}: got {result!r}, expected {expected!r}"
- )
+ assert result == expected.replace(
+ "−", "-"
+ ), f"{d3_format!r} of {value}: got {result!r}, expected {expected!r}"
@pytest.mark.parametrize(