You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/10/15 21:21:37 UTC
(superset) 01/01: feat(chart-data): add opt-in Apache Arrow wire format with per-plugin shim
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch feat/arrow-chart-data-shim
in repository https://gitbox.apache.org/repos/asf/superset.git
commit d7620863fefa502c699c39e45221f3dceb0974f1
Author: Evan Rusackas <ev...@rusackas.com>
AuthorDate: Thu Oct 15 22:20:29 2026 +0100
feat(chart-data): add opt-in Apache Arrow wire format with per-plugin shim
Lets /api/v1/chart/data serve query results as Apache Arrow IPC bytes
instead of JSON, gated per viz type by a new ChartMetadata.supportsArrowFormat
flag so third-party/custom plugins are completely unaffected unless they
explicitly opt in. The frontend parses Arrow transparently into the same
DataRecord[] row shape JSON already produces, so transformProps never sees
the difference.
Backend: adds an Arrow branch to ChartDataRestApi._send_chart_response
mirroring the existing datasource query endpoint's Arrow branch, and lifts
the schema restriction that previously rejected result_format=arrow on this
endpoint.
Frontend: new apache-arrow dependency + arrowBufferToQueryData shim in
@superset-ui/core's query module; ChartMetadata.supportsArrowFormat threaded
through the existing parseMethod-style pattern; exploreJSON resolves
result_format from it; ~24 echarts chart types flip on via the shared
EchartsChartPlugin factory, ~10 other built-in plugins flipped individually.
Benchmark script (scripts/benchmark_arrow_vs_json.py) measures real
JSON-vs-Arrow payload sizes against checked-in example-dataset fixtures.
SQL Lab's own query/results path is explicitly out of scope — no
per-plugin concept to gate on there, and it's architecturally separate.
Co-Authored-By: Evan Rusackas <ev...@preset.io>
Co-Authored-By: Claude Sonnet 5 <no...@anthropic.com>
---
scripts/benchmark_arrow_vs_json.py | 163 +++++++++++++++++
superset-frontend/package-lock.json | 11 ++
.../packages/superset-ui-core/package.json | 1 +
.../src/chart/models/ChartMetadata.ts | 11 ++
.../superset-ui-core/src/query/arrowToQueryData.ts | 197 +++++++++++++++++++++
.../packages/superset-ui-core/src/query/index.ts | 4 +
.../test/chart/models/ChartMetadata.test.ts | 11 ++
.../test/query/arrowToQueryData.test.ts | 146 +++++++++++++++
.../plugin-chart-ag-grid-table/src/index.ts | 1 +
.../plugins/plugin-chart-calendar/src/index.ts | 1 +
.../plugin-chart-cartodiagram/src/plugin/index.ts | 1 +
.../plugins/plugin-chart-chord/src/index.ts | 1 +
.../plugins/plugin-chart-country-map/src/index.ts | 1 +
.../BigNumber/BigNumberPeriodOverPeriod/index.ts | 1 +
.../plugin-chart-echarts/src/Bubble/index.ts | 1 +
.../plugin-chart-echarts/src/Butterfly/index.ts | 1 +
.../plugin-chart-echarts/src/Heatmap/index.ts | 1 +
.../plugin-chart-echarts/src/Histogram/index.ts | 1 +
.../plugin-chart-echarts/src/Sankey/index.ts | 1 +
.../plugin-chart-echarts/src/Waterfall/index.ts | 1 +
.../plugins/plugin-chart-echarts/src/types.ts | 1 +
.../plugin-chart-handlebars/src/plugin/index.ts | 1 +
.../plugins/plugin-chart-horizon/src/index.ts | 1 +
.../plugin-chart-paired-t-test/src/index.ts | 1 +
.../plugin-chart-parallel-coordinates/src/index.ts | 1 +
.../plugins/plugin-chart-partition/src/index.ts | 1 +
.../plugin-chart-pivot-table/src/plugin/index.ts | 1 +
.../plugin-chart-point-cluster-map/src/index.ts | 1 +
.../plugins/plugin-chart-table/src/index.ts | 1 +
.../plugin-chart-word-cloud/src/plugin/index.ts | 1 +
.../plugins/plugin-chart-world-map/src/index.ts | 1 +
.../plugins/preset-chart-deckgl/src/Multi/index.ts | 1 +
.../preset-chart-deckgl/src/layers/Arc/index.ts | 1 +
.../src/layers/Contour/index.ts | 1 +
.../src/layers/Geojson/index.ts | 1 +
.../preset-chart-deckgl/src/layers/Grid/index.ts | 1 +
.../src/layers/Heatmap/index.ts | 1 +
.../preset-chart-deckgl/src/layers/Hex/index.ts | 1 +
.../preset-chart-deckgl/src/layers/Path/index.ts | 1 +
.../src/layers/Polygon/index.ts | 1 +
.../src/layers/Scatter/index.ts | 1 +
.../src/layers/Screengrid/index.ts | 1 +
.../src/components/Chart/chartAction.ts | 39 +++-
.../src/components/Chart/chartActions.test.ts | 83 ++++++++-
.../src/explore/exploreUtils/exploreUtils.test.tsx | 46 +++++
.../src/explore/exploreUtils/index.ts | 17 ++
superset/charts/data/api.py | 19 ++
superset/charts/schemas.py | 22 +--
tests/unit_tests/charts/test_chart_data_api.py | 42 +++++
tests/unit_tests/datasource/test_query_api.py | 15 +-
50 files changed, 834 insertions(+), 27 deletions(-)
diff --git a/scripts/benchmark_arrow_vs_json.py b/scripts/benchmark_arrow_vs_json.py
new file mode 100644
index 00000000000..1ad79438b40
--- /dev/null
+++ b/scripts/benchmark_arrow_vs_json.py
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Benchmark: JSON vs. Arrow IPC payload size for chart-data-shaped DataFrames.
+
+Standalone by design (no Flask app context, no running Superset instance):
+it loads real example-dataset fixtures already checked into the repo under
+``superset/examples/*/data.parquet`` and times/sizes the two serialization
+paths this PR's Arrow wire format sits alongside. This avoids needing a
+running dev server with example data loaded, which may not be available in
+every environment this script is run in.
+
+Both paths reuse the real production serialization code rather than
+hand-rolling a different encoding:
+
+- JSON: ``superset.dataframe.df_to_records`` (the same conversion
+ ``QueryContextProcessor.get_data`` calls for the JSON/table-like result
+ formats) + ``superset.utils.json.dumps(..., default=json_int_dttm_ser,
+ ignore_nan=True)`` (the same call ``ChartDataRestApi._send_chart_response``
+ makes for the JSON branch).
+- Arrow: the exact two-line body of
+ ``QueryContextProcessor._to_arrow_ipc`` (superset/common/query_context_processor.py),
+ inlined here rather than imported, because importing that module pulls in
+ ``superset.models.core`` (encrypted columns, etc.), which requires a Flask
+ app context this script deliberately doesn't start. The logic itself is
+ not duplicated/reinvented: it is copied verbatim and is a 2-line
+ ``pa.Table.from_pandas`` + IPC stream write with no sanitization step.
+
+Usage:
+ PYTHONPATH="$(pwd):$(pwd)/superset-core/src" \\
+ python scripts/benchmark_arrow_vs_json.py
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+import pyarrow as pa
+
+from superset.dataframe import df_to_records
+from superset.utils import json as superset_json
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+EXAMPLES_DIR = REPO_ROOT / "superset" / "examples"
+
+
+def to_arrow_ipc(df: pd.DataFrame) -> bytes:
+ """Verbatim copy of QueryContextProcessor._to_arrow_ipc's body."""
+ table = pa.Table.from_pandas(df, preserve_index=False)
+ sink = pa.BufferOutputStream()
+ with pa.ipc.new_stream(sink, table.schema) as writer:
+ writer.write_table(table)
+ return sink.getvalue().to_pybytes()
+
+
+def json_payload_size(df: pd.DataFrame) -> int:
+ """Same conversion + same json.dumps call _send_chart_response makes for
+ the JSON result format, sized as UTF-8 bytes (what actually goes over
+ the wire)."""
+ records = df_to_records(df)
+ payload = superset_json.dumps(
+ records,
+ default=superset_json.json_int_dttm_ser,
+ ignore_nan=True,
+ )
+ return len(payload.encode("utf-8"))
+
+
+def arrow_payload_size(df: pd.DataFrame) -> int:
+ return len(to_arrow_ipc(df))
+
+
+@dataclass
+class Scenario:
+ label: str
+ df: pd.DataFrame
+
+
+def load_scenarios() -> list[Scenario]:
+ """Representative DataFrames shaped like real example-dataset query
+ results, at a couple of row counts each to show how the gap scales.
+ Real fixtures (not synthetic) are used because they're already checked
+ into the repo and give a genuinely representative column/type mix."""
+ video_game_sales = pd.read_parquet(
+ EXAMPLES_DIR / "video_game_sales" / "data.parquet"
+ )
+ usa_births_names = pd.read_parquet(
+ EXAMPLES_DIR / "usa_births_names" / "data.parquet"
+ )
+ world_health = pd.read_parquet(EXAMPLES_DIR / "world_health" / "data.parquet")
+
+ scenarios = []
+ for n in (500, 5000):
+ scenarios.append(
+ Scenario(
+ f"mixed string+numeric (video_game_sales, {n} rows)",
+ video_game_sales.head(n),
+ )
+ )
+ scenarios.append(
+ Scenario(
+ f"datetime-heavy (usa_births_names, {n} rows)",
+ usa_births_names.head(n),
+ )
+ )
+ scenarios.append(
+ Scenario(
+ f"numeric-heavy, 328 cols (world_health, {n} rows)",
+ world_health.head(n),
+ )
+ )
+ return scenarios
+
+
+def main() -> None:
+ scenarios = load_scenarios()
+ rows = []
+ for scenario in scenarios:
+ json_bytes = json_payload_size(scenario.df)
+ arrow_bytes = arrow_payload_size(scenario.df)
+ pct_change = (arrow_bytes - json_bytes) / json_bytes * 100
+ rows.append((scenario.label, json_bytes, arrow_bytes, pct_change))
+
+ label_width = max(len(r[0]) for r in rows) + 2
+ header = (
+ f"{'scenario':<{label_width}}"
+ f"{'json bytes':>14}{'arrow bytes':>14}{'arrow vs json':>16}"
+ )
+ print(header)
+ print("-" * len(header))
+ for label, json_bytes, arrow_bytes, pct_change in rows:
+ sign = "+" if pct_change >= 0 else ""
+ print(
+ f"{label:<{label_width}}"
+ f"{json_bytes:>14,}{arrow_bytes:>14,}{sign}{pct_change:>14.1f}%"
+ )
+
+
+if __name__ == "__main__":
+ if not os.environ.get("PYTHONPATH"):
+ print(
+ "Warning: PYTHONPATH not set; run with "
+ 'PYTHONPATH="$(pwd):$(pwd)/superset-core/src" to ensure the '
+ "in-repo superset_core package resolves."
+ )
+ main()
diff --git a/superset-frontend/package-lock.json b/superset-frontend/package-lock.json
index abc364980b6..6806ad3a602 100644
--- a/superset-frontend/package-lock.json
+++ b/superset-frontend/package-lock.json
@@ -45544,6 +45544,7 @@
"ace-builds": "^1.44.0",
"ag-grid-community": "36.2.0",
"ag-grid-react": "36.2.0",
+ "apache-arrow": "^21.2.0",
"brace": "^0.11.1",
"classnames": "^2.5.1",
"core-js": "^3.50.0",
@@ -45652,6 +45653,16 @@
"react-dom": ">=16.0.0"
}
},
+ "packages/superset-ui-core/node_modules/apache-arrow/node_modules/@types/node": {
+ "version": "20.19.43",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz",
+ "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==",
+ "extraneous": true,
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~6.21.0"
+ }
+ },
"packages/superset-ui-core/node_modules/d3-format": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz",
diff --git a/superset-frontend/packages/superset-ui-core/package.json b/superset-frontend/packages/superset-ui-core/package.json
index 3d78189dcdf..56f7e5cc49c 100644
--- a/superset-frontend/packages/superset-ui-core/package.json
+++ b/superset-frontend/packages/superset-ui-core/package.json
@@ -59,6 +59,7 @@
"ace-builds": "^1.44.0",
"ag-grid-community": "36.2.0",
"ag-grid-react": "36.2.0",
+ "apache-arrow": "^21.2.0",
"brace": "^0.11.1",
"classnames": "^2.5.1",
"core-js": "^3.50.0",
diff --git a/superset-frontend/packages/superset-ui-core/src/chart/models/ChartMetadata.ts b/superset-frontend/packages/superset-ui-core/src/chart/models/ChartMetadata.ts
index a89ee2e5097..cf3a03792f8 100644
--- a/superset-frontend/packages/superset-ui-core/src/chart/models/ChartMetadata.ts
+++ b/superset-frontend/packages/superset-ui-core/src/chart/models/ChartMetadata.ts
@@ -60,6 +60,13 @@ export interface ChartMetadataConfig {
// opts out — used by time grain/column filters, whose extraFormData is
// not safe to merge across datasets.
supportsCascadeDependencies?: boolean;
+ // Opt-in to the Apache Arrow IPC wire format for this chart type's live
+ // /api/v1/chart/data requests, instead of JSON. The response is still
+ // converted to the same DataRecord[] shape before reaching transformProps
+ // — this does not give the plugin columnar data, only a smaller payload
+ // on the wire. Unset/false (the default) keeps today's JSON behavior, so
+ // third-party plugins that never set this flag are completely unaffected.
+ supportsArrowFormat?: boolean;
}
export default class ChartMetadata {
@@ -107,6 +114,8 @@ export default class ChartMetadata {
supportsCascadeDependencies?: boolean;
+ supportsArrowFormat?: boolean;
+
constructor(config: ChartMetadataConfig) {
const {
name,
@@ -130,6 +139,7 @@ export default class ChartMetadata {
parseMethod = 'json-bigint',
suppressContextMenu = false,
supportsCascadeDependencies,
+ supportsArrowFormat,
} = config;
this.name = name;
@@ -162,6 +172,7 @@ export default class ChartMetadata {
this.parseMethod = parseMethod;
this.suppressContextMenu = suppressContextMenu;
this.supportsCascadeDependencies = supportsCascadeDependencies;
+ this.supportsArrowFormat = supportsArrowFormat;
}
canBeAnnotationType(type: string): boolean {
diff --git a/superset-frontend/packages/superset-ui-core/src/query/arrowToQueryData.ts b/superset-frontend/packages/superset-ui-core/src/query/arrowToQueryData.ts
new file mode 100644
index 00000000000..78df8f9d5f4
--- /dev/null
+++ b/superset-frontend/packages/superset-ui-core/src/query/arrowToQueryData.ts
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { Table, tableFromIPC, Type, util as arrowUtil } from 'apache-arrow';
+import { GenericDataType } from '@apache-superset/core/common';
+import {
+ ChartDataResponseResult,
+ DataRecord,
+ DataRecordValue,
+} from './types/QueryResponse';
+
+/**
+ * Mirrors ``superset/utils/core.py::JS_MAX_INTEGER`` (2**53 - 1). An Arrow
+ * Int64 column decodes to a JS BigInt; integers outside this range travel as
+ * strings on the JSON wire format too (see
+ * ``superset/dataframe.py::_convert_big_integers``), so the Arrow path
+ * matches that here rather than silently truncating precision through
+ * ``Number()``.
+ */
+const JS_MAX_INTEGER = BigInt(Number.MAX_SAFE_INTEGER);
+
+function convertBigInt(value: bigint): DataRecordValue {
+ if (value > JS_MAX_INTEGER || value < -JS_MAX_INTEGER) {
+ return value.toString();
+ }
+ return Number(value);
+}
+
+/**
+ * Decode an Arrow Decimal value (apache-arrow represents it as a
+ * DecimalBigNum wrapping the unscaled integer's raw limbs, not a JS
+ * primitive) into the same JS number the JSON wire format already produces
+ * for a Python ``Decimal`` (``float(value)`` — see
+ * ``superset/utils/json.py::base_json_conv``).
+ *
+ * This loses precision the same way the JSON path already does: both go
+ * through a JS ``number``. Superset's JSON wire format has never carried
+ * exact arbitrary-precision decimals to the browser, so Arrow isn't
+ * regressing anything here.
+ */
+function convertDecimal(value: unknown, scale: number): DataRecordValue {
+ const unscaled = arrowUtil.bigNumToString(
+ value as Parameters<typeof arrowUtil.bigNumToString>[0],
+ );
+ return Number(unscaled) / 10 ** scale;
+}
+
+/**
+ * Non-finite floats (Infinity/-Infinity) round-trip through Arrow as real
+ * IEEE-754 values — ``_to_arrow_ipc`` (superset/common/query_context_processor.py)
+ * does no sanitization, unlike the JSON path's ``df_to_records``, which maps
+ * every non-finite float to ``null`` (superset/dataframe.py). Match the JSON
+ * behavior here so a flagged chart doesn't start seeing real ``Infinity``
+ * values a JSON-format chart never would for the same query. NaN already
+ * arrives as Arrow ``null`` (pyarrow treats float NaN as missing on
+ * ``Table.from_pandas``), so only +/-Infinity needs this guard.
+ */
+function convertNumber(value: number): DataRecordValue {
+ return Number.isFinite(value) ? value : null;
+}
+
+/**
+ * Convert one Arrow column value to the DataRecordValue the JSON wire format
+ * would have produced for the same underlying data.
+ *
+ * Verified empirically against real ``_to_arrow_ipc`` output compared with
+ * ``df_to_records`` + the JSON response's ``json_int_dttm_ser`` for strings,
+ * floats, nulls/NaN, booleans, Date32/Timestamp (incl. tz-aware) columns,
+ * Int64 (both in- and out-of-JS-safe-range), and Decimal128. Timestamps and
+ * dates already decode to plain epoch-millisecond numbers via apache-arrow's
+ * row proxy — identical to ``json_int_dttm_ser`` — so no conversion is
+ * needed for them here.
+ */
+function convertValue(
+ value: unknown,
+ isDecimal: boolean,
+ scale: number,
+): DataRecordValue {
+ if (value === null || value === undefined) {
+ return null;
+ }
+ if (isDecimal) {
+ return convertDecimal(value, scale);
+ }
+ if (typeof value === 'bigint') {
+ return convertBigInt(value);
+ }
+ if (typeof value === 'number') {
+ return convertNumber(value);
+ }
+ // string, boolean: already the right shape.
+ return value as DataRecordValue;
+}
+
+function toGenericDataType(typeId: Type): GenericDataType {
+ switch (typeId) {
+ case Type.Int:
+ case Type.Float:
+ case Type.Decimal:
+ return GenericDataType.Numeric;
+ case Type.Bool:
+ return GenericDataType.Boolean;
+ case Type.Date:
+ case Type.Timestamp:
+ case Type.Time:
+ return GenericDataType.Temporal;
+ case Type.List:
+ case Type.FixedSizeList:
+ case Type.LargeList:
+ return GenericDataType.MultiValue;
+ default:
+ return GenericDataType.String;
+ }
+}
+
+export function arrowTableToDataRecords(table: Table): DataRecord[] {
+ const { fields } = table.schema;
+ const { numRows } = table;
+ const columnMeta = fields.map(field => ({
+ name: field.name,
+ isDecimal: field.type.typeId === Type.Decimal,
+ // Only Decimal types carry a `scale`; TS doesn't narrow `field.type` to
+ // that subtype here, so read it defensively.
+ scale: (field.type as { scale?: number }).scale ?? 0,
+ }));
+
+ return Array.from({ length: numRows }, (_unused, rowIndex) => {
+ const row = table.get(rowIndex);
+ const record: DataRecord = {};
+ columnMeta.forEach(({ name, isDecimal, scale }) => {
+ record[name] = convertValue(row?.[name], isDecimal, scale);
+ });
+ return record;
+ });
+}
+
+/**
+ * Parse an Arrow IPC stream (as returned by the Arrow branches of
+ * ``/api/v1/chart/data`` and ``/api/v1/datasource/<type>/<id>/query``) into
+ * the same ``ChartDataResponseResult`` shape a JSON response's `queries[n]`
+ * entry has, so this is a transparent substitution at the network layer and
+ * every consumer downstream (transformProps, etc.) keeps working unchanged.
+ *
+ * KNOWN GAP: the Arrow IPC bytes carry only the DataFrame itself — none of
+ * the sidecar query metadata (cache_key, is_cached, cached_dttm, the SQL
+ * `query` text, `status`, `stacktrace`, `timing`, `annotation_data`,
+ * `warning`, applied/rejected filters) travels over this wire format today.
+ * Those fields are filled with inert defaults below rather than left
+ * `undefined`, so chrome that reads them (cache badges, "View query", the
+ * timing panel) degrades quietly instead of throwing. Carrying that metadata
+ * over Arrow too is a follow-up, not part of this draft.
+ */
+export function arrowBufferToQueryData(
+ buffer: ArrayBuffer,
+): ChartDataResponseResult {
+ const table = tableFromIPC(buffer);
+ const data = arrowTableToDataRecords(table);
+ const colnames = table.schema.fields.map(field => field.name);
+ const coltypes = table.schema.fields.map(field =>
+ toGenericDataType(field.type.typeId),
+ );
+
+ return {
+ annotation_data: null,
+ cache_key: null,
+ cache_timeout: null,
+ cached_dttm: null,
+ queried_dttm: null,
+ data,
+ colnames,
+ coltypes,
+ error: null,
+ is_cached: false,
+ query: '',
+ rowcount: table.numRows,
+ sql_rowcount: table.numRows,
+ stacktrace: null,
+ status: 'success',
+ from_dttm: null,
+ to_dttm: null,
+ };
+}
diff --git a/superset-frontend/packages/superset-ui-core/src/query/index.ts b/superset-frontend/packages/superset-ui-core/src/query/index.ts
index 9df98aed40e..48bf5e6fe72 100644
--- a/superset-frontend/packages/superset-ui-core/src/query/index.ts
+++ b/superset-frontend/packages/superset-ui-core/src/query/index.ts
@@ -48,3 +48,7 @@ export { default as makeApi } from './api/v1/makeApi';
export { ApiLegacy, ApiV1 };
export { getSemanticSelectionSources } from './semanticSelection';
+export {
+ arrowBufferToQueryData,
+ arrowTableToDataRecords,
+} from './arrowToQueryData';
diff --git a/superset-frontend/packages/superset-ui-core/test/chart/models/ChartMetadata.test.ts b/superset-frontend/packages/superset-ui-core/test/chart/models/ChartMetadata.test.ts
index 78c468bb9c4..d44ae53a33a 100644
--- a/superset-frontend/packages/superset-ui-core/test/chart/models/ChartMetadata.test.ts
+++ b/superset-frontend/packages/superset-ui-core/test/chart/models/ChartMetadata.test.ts
@@ -90,4 +90,15 @@ describe('ChartMetadata', () => {
expect(optedIn.supportsCascadeDependencies).toBe(true);
expect(optedOut.clone().supportsCascadeDependencies).toBe(false);
});
+ test('defaults supportsArrowFormat to falsy, opting a viz type in only when set true', () => {
+ const unset = new ChartMetadata({ name: 'unset', thumbnail: 'test.png' });
+ const optedIn = new ChartMetadata({
+ name: 'opted in',
+ thumbnail: 'test.png',
+ supportsArrowFormat: true,
+ });
+ expect(unset.supportsArrowFormat).toBeFalsy();
+ expect(optedIn.supportsArrowFormat).toBe(true);
+ expect(optedIn.clone().supportsArrowFormat).toBe(true);
+ });
});
diff --git a/superset-frontend/packages/superset-ui-core/test/query/arrowToQueryData.test.ts b/superset-frontend/packages/superset-ui-core/test/query/arrowToQueryData.test.ts
new file mode 100644
index 00000000000..94ffb6b8588
--- /dev/null
+++ b/superset-frontend/packages/superset-ui-core/test/query/arrowToQueryData.test.ts
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { GenericDataType } from '@apache-superset/core/common';
+import { arrowBufferToQueryData } from '@superset-ui/core';
+
+/**
+ * Each fixture below is real Arrow IPC-stream output from Superset's own
+ * ``QueryContextProcessor._to_arrow_ipc`` (superset/common/query_context_processor.py),
+ * captured against a small pandas DataFrame — not hand-rolled Arrow bytes.
+ * The ``expected`` records next to each fixture are what
+ * ``superset/dataframe.py::df_to_records`` plus the JSON response's
+ * ``json.dumps(..., default=json_int_dttm_ser, ignore_nan=True)`` produce
+ * for that *same* DataFrame — i.e. exactly what a JSON-format chart request
+ * would hand to the same plugin's transformProps. This test's job is to
+ * prove arrowBufferToQueryData reproduces those same JS values from the
+ * Arrow bytes, so a plugin can't tell the two wire formats apart.
+ */
+function base64ToArrayBuffer(base64: string): ArrayBuffer {
+ // Built byte-by-byte with this test environment's own Uint8Array/
+ // ArrayBuffer constructors (jsdom), rather than returning Node Buffer's
+ // pooled backing ArrayBuffer — apache-arrow's IPC reader silently reads
+ // zero fields from a cross-realm ArrayBuffer under jsdom.
+ const binary = Buffer.from(base64, 'base64').toString('binary');
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i += 1) {
+ bytes[i] = binary.charCodeAt(i);
+ }
+ return bytes.buffer;
+}
+
+// DataFrame: region (str, w/ null), sales (int64, w/ null -> float64),
+// ratio (float64, incl. NaN), ts (naive datetime64[ns], w/ NaT),
+// flag (bool, w/ null).
+const MIXED_TYPES_AND_NULLS_IPC =
+ '/////ygEAAAQAAAAAAAKAA4ABgAFAAgACgAAAAABBAAQAAAAAAAKAAwAAAAEAAgACgAAAPwCAAAEAAAAAQAAAAwAAAAIAAwABAAIAAgAAADUAgAABAAAAMUCAAB7ImluZGV4X2NvbHVtbnMiOiBbXSwgImNvbHVtbl9pbmRleGVzIjogW10sICJjb2x1bW5zIjogW3sibmFtZSI6ICJyZWdpb24iLCAiZmllbGRfbmFtZSI6ICJyZWdpb24iLCAicGFuZGFzX3R5cGUiOiAidW5pY29kZSIsICJudW1weV90eXBlIjogIm9iamVjdCIsICJtZXRhZGF0YSI6IG51bGx9LCB7Im5hbWUiOiAic2FsZXMiLCAiZmllbGRfbmFtZSI6ICJzYWxlcyIsICJwYW5kYXNfdHlwZSI6ICJmbG9hdDY0IiwgIm51bXB5X3R5cGUiOiAiZmxvYXQ2NCIsICJtZXRhZGF0YSI6IG51b [...]
+
+// DataFrame: big_int (int64, outside +/- JS_MAX_SAFE_INTEGER on rows 1 and 3),
+// small_int (int64, within range), ts_tz (tz-aware datetime64[ns, UTC]).
+const BIGINT_AND_TZ_TIMESTAMP_IPC =
+ '/////zgDAAAQAAAAAAAKAA4ABgAFAAgACgAAAAABBAAQAAAAAAAKAAwAAAAEAAgACgAAADwCAAAEAAAAAQAAAAwAAAAIAAwABAAIAAgAAAAUAgAABAAAAAcCAAB7ImluZGV4X2NvbHVtbnMiOiBbXSwgImNvbHVtbl9pbmRleGVzIjogW10sICJjb2x1bW5zIjogW3sibmFtZSI6ICJiaWdfaW50IiwgImZpZWxkX25hbWUiOiAiYmlnX2ludCIsICJwYW5kYXNfdHlwZSI6ICJpbnQ2NCIsICJudW1weV90eXBlIjogImludDY0IiwgIm1ldGFkYXRhIjogbnVsbH0sIHsibmFtZSI6ICJzbWFsbF9pbnQiLCAiZmllbGRfbmFtZSI6ICJzbWFsbF9pbnQiLCAicGFuZGFzX3R5cGUiOiAiaW50NjQiLCAibnVtcHlfdHlwZSI6ICJpbnQ2NCIsICJtZXRhZGF0YSI6I [...]
+
+// DataFrame: amount (object column of Decimal, precision=4 scale=2), w/ null.
+const DECIMAL_IPC =
+ '/////9ABAAAQAAAAAAAKAA4ABgAFAAgACgAAAAABBAAQAAAAAAAKAAwAAAAEAAgACgAAAFQBAAAEAAAAAQAAAAQAAACA/v//NAEAAAQAAAAlAQAAeyJpbmRleF9jb2x1bW5zIjogW10sICJjb2x1bW5faW5kZXhlcyI6IFtdLCAiY29sdW1ucyI6IFt7Im5hbWUiOiAiYW1vdW50IiwgImZpZWxkX25hbWUiOiAiYW1vdW50IiwgInBhbmRhc190eXBlIjogImRlY2ltYWwiLCAibnVtcHlfdHlwZSI6ICJvYmplY3QiLCAibWV0YWRhdGEiOiB7InByZWNpc2lvbiI6IDQsICJzY2FsZSI6IDJ9fV0sICJhdHRyaWJ1dGVzIjoge30sICJjcmVhdG9yIjogeyJsaWJyYXJ5IjogInB5YXJyb3ciLCAidmVyc2lvbiI6ICIyNS4wLjEifSwgInBhbmRhc192ZXJzaW9uI [...]
+
+// DataFrame: val (float64) = [Infinity, -Infinity, 5.0].
+const INFINITY_IPC =
+ '/////7gBAAAQAAAAAAAKAA4ABgAFAAgACgAAAAABBAAQAAAAAAAKAAwAAAAEAAgACgAAAEABAAAEAAAAAQAAAAwAAAAIAAwABAAIAAgAAAAYAQAABAAAAAgBAAB7ImluZGV4X2NvbHVtbnMiOiBbXSwgImNvbHVtbl9pbmRleGVzIjogW10sICJjb2x1bW5zIjogW3sibmFtZSI6ICJ2YWwiLCAiZmllbGRfbmFtZSI6ICJ2YWwiLCAicGFuZGFzX3R5cGUiOiAiZmxvYXQ2NCIsICJudW1weV90eXBlIjogImZsb2F0NjQiLCAibWV0YWRhdGEiOiBudWxsfV0sICJhdHRyaWJ1dGVzIjoge30sICJjcmVhdG9yIjogeyJsaWJyYXJ5IjogInB5YXJyb3ciLCAidmVyc2lvbiI6ICIyNS4wLjEifSwgInBhbmRhc192ZXJzaW9uIjogIjIuMy4zIn0AAAAABgAAAHBhb [...]
+
+test('converts strings, nulls, NaN, naive timestamps, and booleans to the JSON wire format values', () => {
+ const result = arrowBufferToQueryData(
+ base64ToArrayBuffer(MIXED_TYPES_AND_NULLS_IPC),
+ );
+
+ expect(result.colnames).toEqual(['region', 'sales', 'ratio', 'ts', 'flag']);
+ expect(result.coltypes).toEqual([
+ GenericDataType.String,
+ GenericDataType.Numeric,
+ GenericDataType.Numeric,
+ GenericDataType.Temporal,
+ GenericDataType.Boolean,
+ ]);
+ expect(result.rowcount).toBe(3);
+ expect(result.data).toEqual([
+ {
+ region: 'EMEA',
+ sales: 10,
+ ratio: 1.5,
+ ts: 1704067200000,
+ flag: true,
+ },
+ // pandas' NaN-for-missing-int upcasts the whole `sales` column to
+ // float64; the NaN `ratio` value is pyarrow-nulled on the way in.
+ { region: 'APAC', sales: 20, ratio: null, ts: 1718454600000, flag: false },
+ { region: null, sales: null, ratio: 3.25, ts: null, flag: null },
+ ]);
+});
+
+test('converts Int64 to a number within JS safe-integer range, and to a string outside it', () => {
+ const result = arrowBufferToQueryData(
+ base64ToArrayBuffer(BIGINT_AND_TZ_TIMESTAMP_IPC),
+ );
+
+ expect(result.data).toEqual([
+ {
+ big_int: '123456789012345678', // outside Number.MAX_SAFE_INTEGER
+ small_int: 1,
+ ts_tz: 1704067200000,
+ },
+ { big_int: 42, small_int: 2, ts_tz: 1718454600000 },
+ {
+ big_int: '-987654321098765432', // outside -Number.MAX_SAFE_INTEGER
+ small_int: 3,
+ ts_tz: 1735689599000,
+ },
+ ]);
+});
+
+test('decodes Decimal128 the same way the JSON path converts a Python Decimal (float())', () => {
+ const result = arrowBufferToQueryData(base64ToArrayBuffer(DECIMAL_IPC));
+
+ expect(result.coltypes).toEqual([GenericDataType.Numeric]);
+ expect(result.data).toEqual([
+ { amount: 10.5 },
+ { amount: 20.25 },
+ { amount: null },
+ ]);
+});
+
+test('nulls out +/-Infinity to match df_to_records, even though pyarrow does not', () => {
+ // This is the one real divergence found between the two wire formats:
+ // _to_arrow_ipc has no non-finite sanitization (unlike df_to_records), so
+ // the raw Arrow bytes here really do contain IEEE-754 Infinity/-Infinity.
+ // arrowBufferToQueryData corrects for it so a flagged chart can't observe
+ // real Infinity values a JSON-format chart never would for the same data.
+ const result = arrowBufferToQueryData(base64ToArrayBuffer(INFINITY_IPC));
+
+ expect(result.data).toEqual([{ val: null }, { val: null }, { val: 5 }]);
+});
+
+test('fills in inert defaults for sidecar metadata the Arrow transport does not carry', () => {
+ const result = arrowBufferToQueryData(
+ base64ToArrayBuffer(MIXED_TYPES_AND_NULLS_IPC),
+ );
+
+ expect(result.cache_key).toBeNull();
+ expect(result.is_cached).toBe(false);
+ expect(result.error).toBeNull();
+ expect(result.status).toBe('success');
+});
diff --git a/superset-frontend/plugins/plugin-chart-ag-grid-table/src/index.ts b/superset-frontend/plugins/plugin-chart-ag-grid-table/src/index.ts
index 085f22ae108..c4dc2978bc0 100644
--- a/superset-frontend/plugins/plugin-chart-ag-grid-table/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-ag-grid-table/src/index.ts
@@ -39,6 +39,7 @@ export {
} from './stateConversion';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
behaviors: [
Behavior.InteractiveChart,
Behavior.DrillToDetail,
diff --git a/superset-frontend/plugins/plugin-chart-calendar/src/index.ts b/superset-frontend/plugins/plugin-chart-calendar/src/index.ts
index fb326c7a5d2..6fb4cb1602d 100644
--- a/superset-frontend/plugins/plugin-chart-calendar/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-calendar/src/index.ts
@@ -26,6 +26,7 @@ import thumbnail from './images/thumbnail.png';
import thumbnailDark from './images/thumbnail-dark.png';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Correlation'),
credits: ['https://github.com/wa0x6e/cal-heatmap'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-cartodiagram/src/plugin/index.ts b/superset-frontend/plugins/plugin-chart-cartodiagram/src/plugin/index.ts
index 6f3cd1ff429..6751705d6e0 100644
--- a/superset-frontend/plugins/plugin-chart-cartodiagram/src/plugin/index.ts
+++ b/superset-frontend/plugins/plugin-chart-cartodiagram/src/plugin/index.ts
@@ -33,6 +33,7 @@ import { getLayerConfig } from '../util/controlPanelUtil';
export default class CartodiagramPlugin extends ChartPlugin {
constructor(opts: CartodiagramPluginConstructorOpts) {
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
description: t(
'Display charts on a map. For using this plugin, users first have to create any other chart that can then be placed on the map.',
),
diff --git a/superset-frontend/plugins/plugin-chart-chord/src/index.ts b/superset-frontend/plugins/plugin-chart-chord/src/index.ts
index 671dfd0e07f..6d23d9ff96d 100644
--- a/superset-frontend/plugins/plugin-chart-chord/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-chord/src/index.ts
@@ -26,6 +26,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Flow'),
credits: ['https://github.com/d3/d3-chord'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-country-map/src/index.ts b/superset-frontend/plugins/plugin-chart-country-map/src/index.ts
index dc402fa8545..f414197bf7a 100644
--- a/superset-frontend/plugins/plugin-chart-country-map/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-country-map/src/index.ts
@@ -28,6 +28,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://bl.ocks.org/john-guerra'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/index.ts
index fc4555c554d..eb01c6bf048 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/index.ts
@@ -27,6 +27,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
export default class PopKPIPlugin extends ChartPlugin {
constructor() {
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('KPI'),
description: t(
'Showcases a metric along with a comparison of value, change, and percent change for a selected time period.',
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Bubble/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Bubble/index.ts
index 35b852ec165..2bb9619ad3e 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Bubble/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Bubble/index.ts
@@ -39,6 +39,7 @@ export default class EchartsBubbleChartPlugin extends ChartPlugin<
controlPanel,
loadChart: () => import('./EchartsBubble'),
metadata: new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Correlation'),
credits: ['https://echarts.apache.org'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Butterfly/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Butterfly/index.ts
index cc94ec201e0..1611bdfbaa8 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Butterfly/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Butterfly/index.ts
@@ -37,6 +37,7 @@ export default class EchartsButterflyChartPlugin extends ChartPlugin<
controlPanel,
loadChart: () => import('./Butterfly'),
metadata: new ChartMetadata({
+ supportsArrowFormat: true,
behaviors: [
Behavior.InteractiveChart,
Behavior.DrillToDetail,
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Heatmap/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Heatmap/index.ts
index 56965454a11..455878c45dc 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Heatmap/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Heatmap/index.ts
@@ -31,6 +31,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Correlation'),
description: t(
'Visualize a related metric across pairs of groups. Heatmaps excel at showcasing the correlation or strength between two groups. Color is used to emphasize the strength of the link between each pair of groups.',
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Histogram/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Histogram/index.ts
index f0e0d8905b8..d584a4a753a 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Histogram/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Histogram/index.ts
@@ -51,6 +51,7 @@ export default class EchartsHistogramChartPlugin extends ChartPlugin<
controlPanel,
loadChart: () => import('./Histogram'),
metadata: new ChartMetadata({
+ supportsArrowFormat: true,
credits: ['https://echarts.apache.org'],
category: t('Distribution'),
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Sankey/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Sankey/index.ts
index e473bf096b4..e7d75816370 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Sankey/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Sankey/index.ts
@@ -51,6 +51,7 @@ export default class EchartsSankeyChartPlugin extends ChartPlugin<
controlPanel,
loadChart: () => import('./Sankey'),
metadata: new ChartMetadata({
+ supportsArrowFormat: true,
credits: ['https://echarts.apache.org'],
category: t('Flow'),
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Waterfall/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Waterfall/index.ts
index 7a67fc90e2a..c0312b2a6d7 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/Waterfall/index.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/Waterfall/index.ts
@@ -53,6 +53,7 @@ export default class EchartsWaterfallChartPlugin extends ChartPlugin<
controlPanel,
loadChart: () => import('./EchartsWaterfall'),
metadata: new ChartMetadata({
+ supportsArrowFormat: true,
credits: ['https://echarts.apache.org'],
category: t('Evolution'),
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/types.ts b/superset-frontend/plugins/plugin-chart-echarts/src/types.ts
index a5c54541785..8d6d9acf720 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/types.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/types.ts
@@ -197,6 +197,7 @@ export class EchartsChartPlugin<
...restProps,
metadata: new ChartMetadata({
parseMethod: 'json',
+ supportsArrowFormat: true,
...metadata,
}),
});
diff --git a/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/index.ts b/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/index.ts
index f8e28f2ad31..94d2101e328 100644
--- a/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/index.ts
+++ b/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/index.ts
@@ -41,6 +41,7 @@ export default class HandlebarsChartPlugin extends ChartPlugin {
*/
constructor() {
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
description: t('Write a handlebars template to render the data'),
name: t('Handlebars'),
thumbnail,
diff --git a/superset-frontend/plugins/plugin-chart-horizon/src/index.ts b/superset-frontend/plugins/plugin-chart-horizon/src/index.ts
index 034e9775d73..ba5a8dc38a6 100644
--- a/superset-frontend/plugins/plugin-chart-horizon/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-horizon/src/index.ts
@@ -26,6 +26,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Distribution'),
credits: ['http://kmandov.github.io/d3-horizon-chart/'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-paired-t-test/src/index.ts b/superset-frontend/plugins/plugin-chart-paired-t-test/src/index.ts
index 29ad81a22ee..bc4f2fc83b1 100644
--- a/superset-frontend/plugins/plugin-chart-paired-t-test/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-paired-t-test/src/index.ts
@@ -26,6 +26,7 @@ import thumbnailDark from './images/thumbnail-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Correlation'),
description: t(
'Table that visualizes paired t-tests, which are used to understand statistical differences between groups.',
diff --git a/superset-frontend/plugins/plugin-chart-parallel-coordinates/src/index.ts b/superset-frontend/plugins/plugin-chart-parallel-coordinates/src/index.ts
index c0ea250932d..c9f050ecdfe 100644
--- a/superset-frontend/plugins/plugin-chart-parallel-coordinates/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-parallel-coordinates/src/index.ts
@@ -28,6 +28,7 @@ import example2Dark from './images/example2-dark.jpg';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Ranking'),
credits: ['https://syntagmatic.github.io/parallel-coordinates'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-partition/src/index.ts b/superset-frontend/plugins/plugin-chart-partition/src/index.ts
index cb75e56f401..fd7f1c2c3da 100644
--- a/superset-frontend/plugins/plugin-chart-partition/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-partition/src/index.ts
@@ -26,6 +26,7 @@ import exampleDark from './images/example-dark.jpg';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Part of a Whole'),
description: t('Compare the same summarized metric across multiple groups.'),
exampleGallery: [{ url: example, urlDark: exampleDark }],
diff --git a/superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/index.ts b/superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/index.ts
index 4108faa0c59..ac1ca784c69 100644
--- a/superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/index.ts
+++ b/superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/index.ts
@@ -49,6 +49,7 @@ export default class PivotTableChartPlugin extends ChartPlugin<
*/
constructor() {
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
behaviors: [
Behavior.InteractiveChart,
Behavior.DrillToDetail,
diff --git a/superset-frontend/plugins/plugin-chart-point-cluster-map/src/index.ts b/superset-frontend/plugins/plugin-chart-point-cluster-map/src/index.ts
index 1f95c36a98d..64af8e9e595 100644
--- a/superset-frontend/plugins/plugin-chart-point-cluster-map/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-point-cluster-map/src/index.ts
@@ -27,6 +27,7 @@ import example2Dark from './images/MapBox2-dark.jpg';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://maplibre.org/'],
description: '',
diff --git a/superset-frontend/plugins/plugin-chart-table/src/index.ts b/superset-frontend/plugins/plugin-chart-table/src/index.ts
index 2b1d72b0b24..0704170278d 100644
--- a/superset-frontend/plugins/plugin-chart-table/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-table/src/index.ts
@@ -34,6 +34,7 @@ import { TableChartFormData, TableChartProps } from './types';
export * from './types';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
behaviors: [
Behavior.InteractiveChart,
Behavior.DrillToDetail,
diff --git a/superset-frontend/plugins/plugin-chart-word-cloud/src/plugin/index.ts b/superset-frontend/plugins/plugin-chart-word-cloud/src/plugin/index.ts
index e1977db2a8f..9ca46968a26 100644
--- a/superset-frontend/plugins/plugin-chart-word-cloud/src/plugin/index.ts
+++ b/superset-frontend/plugins/plugin-chart-word-cloud/src/plugin/index.ts
@@ -31,6 +31,7 @@ import example2Dark from '../images/Word_Cloud_2-dark.jpg';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Ranking'),
credits: ['https://github.com/jasondavies/d3-cloud'],
description: t(
diff --git a/superset-frontend/plugins/plugin-chart-world-map/src/index.ts b/superset-frontend/plugins/plugin-chart-world-map/src/index.ts
index f1036908795..f23b1093cff 100644
--- a/superset-frontend/plugins/plugin-chart-world-map/src/index.ts
+++ b/superset-frontend/plugins/plugin-chart-world-map/src/index.ts
@@ -28,6 +28,7 @@ import example2Dark from './images/WorldMap2-dark.jpg';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['http://datamaps.github.io/'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/Multi/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/Multi/index.ts
index df4e2fa04a7..48fb32a6f77 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/Multi/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/Multi/index.ts
@@ -26,6 +26,7 @@ import transformProps from '../transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t('Compose multiple layers together to form complex visuals.'),
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Arc/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Arc/index.ts
index e077e529f6a..7f0f9a3eba9 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Arc/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Arc/index.ts
@@ -27,6 +27,7 @@ import buildQuery from './buildQuery';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
behaviors: [
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Contour/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Contour/index.ts
index 5fa660e8d04..37fff1575d7 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Contour/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Contour/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Geojson/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Geojson/index.ts
index 618beff2b46..222a4109a20 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Geojson/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Geojson/index.ts
@@ -25,6 +25,7 @@ import exampleDark from './images/example-dark.png';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Grid/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Grid/index.ts
index 8055b8b716c..d17141ef02c 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Grid/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Grid/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Heatmap/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Heatmap/index.ts
index 465d9f740f0..6c806cd49e6 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Heatmap/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Heatmap/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Hex/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Hex/index.ts
index 0c95e97a2b7..a41216c4b97 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Hex/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Hex/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Path/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Path/index.ts
index e2e64d553f8..b884ad40ce6 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Path/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Path/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t('Visualizes connected points, which form a path, on a map.'),
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Polygon/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Polygon/index.ts
index 0d5a691ab4e..9c51de1d323 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Polygon/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Polygon/index.ts
@@ -27,6 +27,7 @@ import buildQuery from './buildQuery';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Scatter/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Scatter/index.ts
index ee43b4a2749..a16feba79e9 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Scatter/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Scatter/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Screengrid/index.ts b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Screengrid/index.ts
index 30c6d9d7f67..4e51b1dd991 100644
--- a/superset-frontend/plugins/preset-chart-deckgl/src/layers/Screengrid/index.ts
+++ b/superset-frontend/plugins/preset-chart-deckgl/src/layers/Screengrid/index.ts
@@ -27,6 +27,7 @@ import transformProps from './transformProps';
import controlPanel from './controlPanel';
const metadata = new ChartMetadata({
+ supportsArrowFormat: true,
category: t('Map'),
credits: ['https://uber.github.io/deck.gl'],
description: t(
diff --git a/superset-frontend/src/components/Chart/chartAction.ts b/superset-frontend/src/components/Chart/chartAction.ts
index 83d69dd37c2..c002acd8f38 100644
--- a/superset-frontend/src/components/Chart/chartAction.ts
+++ b/superset-frontend/src/components/Chart/chartAction.ts
@@ -30,6 +30,7 @@ import {
DataMask,
DatasourceType,
LatestQueryFormData,
+ arrowBufferToQueryData,
} from '@superset-ui/core';
import { t } from '@apache-superset/core/translation';
import type { ControlStateMapping } from '@superset-ui/chart-controls';
@@ -39,6 +40,7 @@ import {
buildV1ChartDataPayload,
getQuerySettings,
getChartDataUri,
+ supportsArrowResultFormat,
} from 'src/explore/exploreUtils';
import {
addDangerToast,
@@ -484,6 +486,28 @@ const v1ChartDataRequest = async (
parseMethod,
};
+ if (resultFormat === 'arrow') {
+ // Arrow responses aren't JSON, so SupersetClient must not try to parse
+ // them as such: 'raw' hands back the untouched Response (see
+ // getChartDataRequest, which forces parseMethod to 'raw' whenever
+ // resultFormat is 'arrow', regardless of this viz type's own
+ // ChartMetadata.parseMethod). Only a chart type that opted in via
+ // ChartMetadata.supportsArrowFormat ever reaches this branch; async
+ // mode is never requested for Arrow (see getChartDataRequest), so the
+ // response here is always either the final result or an HTTP error
+ // that SupersetClient already rejected before this line.
+ const response = (await SupersetClient.post(
+ querySettings as Parameters<typeof SupersetClient.post>[0],
+ )) as unknown as Response;
+ const buffer = await response.arrayBuffer();
+ // Converted to the same ChartDataResponseResult shape a JSON response's
+ // `queries[0]` would have, so every consumer downstream of this request
+ // (extractResult, handleChartDataResponse, transformProps) keeps working
+ // unchanged — the Arrow transport is invisible past this point.
+ const queryResult = arrowBufferToQueryData(buffer);
+ return { response, json: { result: [queryResult] } };
+ }
+
return SupersetClient.post(
querySettings as Parameters<typeof SupersetClient.post>[0],
) as Promise<ChartDataRequestResponse>;
@@ -525,6 +549,10 @@ export async function getChartDataRequest({
};
}
const [parseMethod] = getQuerySettings(formData);
+ // Arrow bytes must never be run through SupersetClient's JSON parsing,
+ // regardless of what this viz type's own ChartMetadata.parseMethod says
+ // (that field exists for json-bigint vs. json, a JSON-vs-JSON choice).
+ const effectiveParseMethod = resultFormat === 'arrow' ? 'raw' : parseMethod;
return v1ChartDataRequest(
formData,
resultFormat,
@@ -533,7 +561,7 @@ export async function getChartDataRequest({
querySettings,
setDataMask,
ownState,
- parseMethod,
+ effectiveParseMethod,
asyncMode,
queryForceNonces,
);
@@ -813,11 +841,18 @@ export function exploreJSON(
setTimeout(() => prevController.abort(), 0);
}
+ // Only a chart type that opted in via ChartMetadata.supportsArrowFormat
+ // ever requests Arrow; every other viz type keeps getting 'json' exactly
+ // as before. This is the only call site that decides the wire format for
+ // a chart's live render — exports (exportChart) and the "View query" /
+ // SQL Lab redirect (redirectSQLLab) always request 'json' explicitly and
+ // are untouched.
+ const resultFormat = supportsArrowResultFormat(formData) ? 'arrow' : 'json';
const chartDataRequestCaught = requestChartDataResolved(
{
setDataMask,
formData,
- resultFormat: 'json',
+ resultFormat,
resultType: 'full',
force,
requestParams,
diff --git a/superset-frontend/src/components/Chart/chartActions.test.ts b/superset-frontend/src/components/Chart/chartActions.test.ts
index 6f8bb4e17f8..d59c66552d1 100644
--- a/superset-frontend/src/components/Chart/chartActions.test.ts
+++ b/superset-frontend/src/components/Chart/chartActions.test.ts
@@ -104,7 +104,7 @@ describe('chart actions', () => {
let getChartDataUriStub: jest.SpyInstance;
let buildV1ChartDataPayloadStub: jest.SpyInstance;
let waitForAsyncDataStub: jest.SpyInstance;
- let fakeMetadata: { viz_type?: string };
+ let fakeMetadata: { viz_type?: string; supportsArrowFormat?: boolean };
beforeAll(() => {
fetchMock.get('glob:*api/v1/security/csrf_token/*', { result: '1234' });
@@ -358,6 +358,87 @@ describe('chart actions', () => {
expect(dispatch.mock.calls[0][0].type).toBe(actions.CHART_UPDATE_STARTED);
});
+ test('requests Arrow and converts the raw response to row dicts for an opted-in viz type', async () => {
+ const { tableFromArrays, tableToIPC } = jest.requireActual(
+ 'apache-arrow',
+ ) as typeof import('apache-arrow');
+ const arrowBytes: Uint8Array = tableToIPC(
+ tableFromArrays({ region: ['EMEA', 'APAC'], sales: [10, 20] }),
+ 'stream',
+ );
+ // Mocked one level below the network: this still exercises the real
+ // v1ChartDataRequest Arrow branch (response.arrayBuffer() ->
+ // arrowBufferToQueryData), without relying on fetch-mock/jsdom's
+ // fetch polyfill to carry binary bytes faithfully across realms,
+ // which isn't representative of a real single-realm browser anyway.
+ const postSpy = jest.spyOn(SupersetClient, 'post').mockResolvedValue({
+ status: 200,
+ arrayBuffer: () => Promise.resolve(arrowBytes.buffer),
+ } as unknown as Response);
+
+ // Simulates a viz type whose ChartMetadata sets supportsArrowFormat.
+ fakeMetadata = { viz_type: 'my_viz', supportsArrowFormat: true };
+ buildV1ChartDataPayloadStub.mockResolvedValue({
+ some_param: 'fake query!',
+ result_type: 'full',
+ result_format: 'arrow',
+ } as unknown as Awaited<
+ ReturnType<typeof exploreUtils.buildV1ChartDataPayload>
+ >);
+
+ const chartKey = 'arrow_test';
+ const dispatchMock = jest.fn();
+ const updateDataMaskSpy = jest
+ .spyOn(dataMaskActions, 'updateDataMask')
+ .mockReturnValue({ type: 'UPDATE_DATA_MASK' } as ReturnType<
+ typeof dataMaskActions.updateDataMask
+ >);
+
+ try {
+ const thunkAction = actions.exploreJSON(
+ { viz_type: 'my_viz' } as QueryFormData,
+ false,
+ undefined,
+ chartKey,
+ );
+ await thunkAction(
+ dispatchMock as unknown as actions.ChartThunkDispatch,
+ mockGetState as unknown as () => actions.RootState,
+ undefined,
+ );
+
+ // The request body still carries result_format: 'arrow' — confirms
+ // exploreJSON actually resolved it from ChartMetadata rather than
+ // hardcoding 'json'. parseMethod is forced to 'raw' regardless of
+ // this viz type's own ChartMetadata.parseMethod.
+ expect(postSpy).toHaveBeenCalledTimes(1);
+ const requestConfig = postSpy.mock.calls[0][0] as unknown as {
+ body: string;
+ parseMethod?: string;
+ };
+ expect(JSON.parse(requestConfig.body)).toEqual({
+ some_param: 'fake query!',
+ result_type: 'full',
+ result_format: 'arrow',
+ });
+ expect(requestConfig.parseMethod).toBe('raw');
+
+ const succeeded = dispatchMock.mock.calls.find(
+ ([action]) => action?.type === actions.CHART_UPDATE_SUCCEEDED,
+ );
+ expect(succeeded).toBeDefined();
+ const [queriesResponse] = succeeded![0].queriesResponse;
+ expect(queriesResponse.data).toEqual([
+ { region: 'EMEA', sales: 10 },
+ { region: 'APAC', sales: 20 },
+ ]);
+ expect(queriesResponse.colnames).toEqual(['region', 'sales']);
+ } finally {
+ updateDataMaskSpy.mockRestore();
+ postSpy.mockRestore();
+ }
+ });
+
test('should handle the bigint without regression', async () => {
getChartDataUriStub.mockRestore();
const mockBigIntUrl = '/mock/chart/data/bigint';
diff --git a/superset-frontend/src/explore/exploreUtils/exploreUtils.test.tsx b/superset-frontend/src/explore/exploreUtils/exploreUtils.test.tsx
index 9dffbe29e0a..97637640ad9 100644
--- a/superset-frontend/src/explore/exploreUtils/exploreUtils.test.tsx
+++ b/superset-frontend/src/explore/exploreUtils/exploreUtils.test.tsx
@@ -24,6 +24,7 @@ import {
getSimpleSQLExpression,
getQuerySettings,
mountExploreUrl,
+ supportsArrowResultFormat,
} from 'src/explore/exploreUtils';
import { DashboardStandaloneMode } from 'src/dashboard/util/constants';
import * as hostNamesConfig from 'src/utils/hostNamesConfig';
@@ -257,6 +258,51 @@ describe('exploreUtils', () => {
});
});
+ // eslint-disable-next-line no-restricted-globals -- TODO: Migrate from describe blocks
+ describe('supportsArrowResultFormat', () => {
+ beforeAll(() => {
+ getChartMetadataRegistry().registerValue('my_custom_arrow_viz', {
+ supportsArrowFormat: true,
+ } as unknown as ChartMetadata);
+ getChartMetadataRegistry().registerValue('my_custom_non_arrow_viz', {
+ supportsArrowFormat: false,
+ } as unknown as ChartMetadata);
+ });
+
+ afterAll(() => {
+ getChartMetadataRegistry().remove('my_custom_arrow_viz');
+ getChartMetadataRegistry().remove('my_custom_non_arrow_viz');
+ });
+
+ test('is true only for a viz type that opted in', () => {
+ expect(
+ supportsArrowResultFormat({
+ ...formData,
+ viz_type: 'my_custom_arrow_viz',
+ }),
+ ).toBe(true);
+ });
+
+ test('is false for a viz type that explicitly left it unset/false', () => {
+ expect(
+ supportsArrowResultFormat({
+ ...formData,
+ viz_type: 'my_custom_non_arrow_viz',
+ }),
+ ).toBe(false);
+ });
+
+ test('defaults to false for an unregistered viz type', () => {
+ expect(
+ supportsArrowResultFormat({ ...formData, viz_type: 'undefined_viz' }),
+ ).toBe(false);
+ });
+
+ test('defaults to false without a viz_type', () => {
+ expect(supportsArrowResultFormat(formData)).toBe(false);
+ });
+ });
+
// eslint-disable-next-line no-restricted-globals -- TODO: Migrate from describe blocks
describe('getSimpleSQLExpression', () => {
test('returns empty string when subject is undefined', () => {
diff --git a/superset-frontend/src/explore/exploreUtils/index.ts b/superset-frontend/src/explore/exploreUtils/index.ts
index 30b9da533df..a9ada79466d 100644
--- a/superset-frontend/src/explore/exploreUtils/index.ts
+++ b/superset-frontend/src/explore/exploreUtils/index.ts
@@ -287,6 +287,23 @@ export const getQuerySettings = (
return [vizMetadata?.parseMethod ?? 'json-bigint'];
};
+/**
+ * Whether this chart type opted into the Apache Arrow wire format for its
+ * live /api/v1/chart/data requests (ChartMetadata.supportsArrowFormat). This
+ * is the entire compatibility mechanism: a viz plugin that never sets the
+ * flag always gets back `false` here, so it keeps requesting (and parsing)
+ * JSON exactly as before — this function, and everything gated on it, is a
+ * complete no-op for third-party/custom plugins.
+ */
+export const supportsArrowResultFormat = (
+ formData: Partial<QueryFormData>,
+): boolean => {
+ const vizMetadata = formData.viz_type
+ ? getChartMetadataRegistry().get(formData.viz_type)
+ : undefined;
+ return Boolean(vizMetadata?.supportsArrowFormat);
+};
+
export const buildV1ChartDataPayload = async ({
formData,
force,
diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py
index 5dd2b41168c..229b2c74f01 100644
--- a/superset/charts/data/api.py
+++ b/superset/charts/data/api.py
@@ -474,6 +474,25 @@ class ChartDataRestApi(ChartRestApi):
datasource,
)
+ if result_format == ChartDataResultFormat.ARROW:
+ # Arrow is a live wire format, not a downloadable export, so it's
+ # handled as its own branch rather than folded into table_like()
+ # (which gates the CSV/XLSX export permission check below). This
+ # mirrors the datasource query endpoint's Arrow branch
+ # (``superset/datasource/api.py::DatasourceRestApi._run``). Only a
+ # viz plugin that opts in via ``ChartMetadata.supportsArrowFormat``
+ # ever requests this format, so third-party plugins that never set
+ # the flag are unaffected.
+ queries = materialized_result["queries"]
+ if len(queries) != 1:
+ return self.response_400(
+ message="Arrow result format supports exactly one query."
+ )
+ return Response(
+ queries[0]["data"],
+ mimetype="application/vnd.apache.arrow.stream",
+ )
+
if result_format in ChartDataResultFormat.table_like():
# Verify user has permission to export file
if not self._has_export_permission():
diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py
index a4c90628b75..efb6603ae0a 100644
--- a/superset/charts/schemas.py
+++ b/superset/charts/schemas.py
@@ -32,7 +32,7 @@ from marshmallow import (
validates_schema,
ValidationError,
)
-from marshmallow.validate import Length, NoneOf, Range
+from marshmallow.validate import Length, Range
from marshmallow_union import Union
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
@@ -1699,21 +1699,11 @@ class ChartDataQueryContextSchema(Schema):
)
result_type = fields.Enum(ChartDataResultType, by_value=True)
- result_format = fields.Enum(
- ChartDataResultFormat,
- by_value=True,
- # Arrow is served only by the datasource query endpoint;
- # ``_send_chart_response`` has no Arrow branch. Rejecting it here fails
- # fast, rather than executing the query and only then returning
- # "Unsupported result_format".
- validate=NoneOf(
- [ChartDataResultFormat.ARROW],
- error=(
- "result_format 'arrow' is not supported by this endpoint; use "
- "POST /api/v1/datasource/<type>/<id>/query."
- ),
- ),
- )
+ # Arrow is an opt-in wire format for this endpoint: a viz plugin's
+ # ``ChartMetadata.supportsArrowFormat`` gates whether the frontend ever
+ # sends it, and ``_send_chart_response`` has a matching Arrow branch that
+ # mirrors the datasource query endpoint's.
+ result_format = fields.Enum(ChartDataResultFormat, by_value=True)
form_data = fields.Raw(allow_none=True, required=False)
diff --git a/tests/unit_tests/charts/test_chart_data_api.py b/tests/unit_tests/charts/test_chart_data_api.py
index a86d4bf75df..d5615fedab3 100644
--- a/tests/unit_tests/charts/test_chart_data_api.py
+++ b/tests/unit_tests/charts/test_chart_data_api.py
@@ -1058,6 +1058,48 @@ def test_send_chart_response_does_not_project_timing_for_csv(
assert "timing" not in query_payload
+def test_send_chart_response_arrow_returns_single_query_bytes() -> None:
+ """Mirrors the datasource query endpoint's Arrow branch
+ (DatasourceRestApi._run): a single-query Arrow result is returned as the
+ raw Arrow IPC bytes with the Arrow stream mimetype, not wrapped in JSON.
+ """
+ query_context = MagicMock()
+ query_context.result_type = ChartDataResultType.FULL
+ query_context.result_format = ChartDataResultFormat.ARROW
+
+ arrow_bytes = b"\x00\x01\x02arrow-ipc-payload"
+ result = {
+ "query_context": query_context,
+ "queries": [{"data": arrow_bytes}],
+ }
+
+ api = ChartDataRestApi()
+ response = api._send_chart_response(result)
+
+ assert response.status_code == 200
+ assert response.mimetype == "application/vnd.apache.arrow.stream"
+ assert response.get_data() == arrow_bytes
+
+
+def test_send_chart_response_arrow_rejects_multiple_queries() -> None:
+ """Arrow IPC bytes describe a single table; a multi-query chart has no
+ well-defined single-stream representation, so it's a 400 rather than
+ silently returning only the first query's bytes."""
+ query_context = MagicMock()
+ query_context.result_type = ChartDataResultType.FULL
+ query_context.result_format = ChartDataResultFormat.ARROW
+
+ result = {
+ "query_context": query_context,
+ "queries": [{"data": b"query-1"}, {"data": b"query-2"}],
+ }
+
+ api = ChartDataRestApi()
+ response = api._send_chart_response(result)
+
+ assert response.status_code == 400
+
+
def _extract_filename(form_value: str) -> str | None:
"""Run _extract_export_params_from_request with a form filename value."""
app = Flask(__name__)
diff --git a/tests/unit_tests/datasource/test_query_api.py b/tests/unit_tests/datasource/test_query_api.py
index a20510eda69..21e06d9908b 100644
--- a/tests/unit_tests/datasource/test_query_api.py
+++ b/tests/unit_tests/datasource/test_query_api.py
@@ -173,19 +173,18 @@ def test_arrow_serializer_round_trips() -> None:
pd.testing.assert_frame_equal(restored, df)
-def test_chart_data_schema_rejects_arrow() -> None:
- """`ARROW` lives on the shared enum so `get_data` can serialize it, but
- `_send_chart_response` has no Arrow branch — accepting it on chart/data
- would execute the query and only then fail with "Unsupported result_format".
+def test_chart_data_schema_accepts_arrow() -> None:
+ """`ARROW` is an opt-in wire format for /api/v1/chart/data too: a viz
+ plugin's `ChartMetadata.supportsArrowFormat` gates whether the frontend
+ ever sends it, and `_send_chart_response` has a matching Arrow branch
+ (mirroring the datasource query endpoint's) rather than falling through
+ to "Unsupported result_format".
"""
from superset.charts.schemas import ChartDataQueryContextSchema
field = ChartDataQueryContextSchema().fields["result_format"]
- with pytest.raises(ValidationError) as excinfo:
- field.deserialize("arrow")
- assert "arrow" in str(excinfo.value)
- assert "/api/v1/datasource/" in str(excinfo.value)
+ assert field.deserialize("arrow") == ChartDataResultFormat.ARROW
# Its own formats are unaffected.
for fmt in ("json", "csv", "xlsx"):