You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2025/03/14 00:09:47 UTC
(superset) 01/01: add more csv tests
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch elizabeth/csv-tests
in repository https://gitbox.apache.org/repos/asf/superset.git
commit c48d1c9ab1d560460482d866c65a1393f38c4181
Author: Elizabeth Thompson <es...@gmail.com>
AuthorDate: Thu Mar 13 17:09:27 2025 -0700
add more csv tests
---
tests/unit_tests/charts/test_client_processing.py | 91 ++++++++++
tests/unit_tests/utils/csv_tests.py | 208 +++++++++++++++++++++-
2 files changed, 293 insertions(+), 6 deletions(-)
diff --git a/tests/unit_tests/charts/test_client_processing.py b/tests/unit_tests/charts/test_client_processing.py
index b38513a2e3..cecc43c45b 100644
--- a/tests/unit_tests/charts/test_client_processing.py
+++ b/tests/unit_tests/charts/test_client_processing.py
@@ -2060,6 +2060,97 @@ COUNT(is_software_dev)
}
+def test_apply_client_processing_csv_format_simple_table():
+ """
+ It should be able to process csv results
+ And not show a default column
+ """
+
+ result = {
+ "queries": [
+ {
+ "result_format": ChartDataResultFormat.CSV,
+ "data": """
+COUNT(is_software_dev)
+4725
+""",
+ }
+ ]
+ }
+ form_data = {
+ "datasource": "19__table",
+ "viz_type": "table",
+ "slice_id": 69,
+ "url_params": {},
+ "granularity_sqla": "time_start",
+ "time_grain_sqla": "P1D",
+ "time_range": "No filter",
+ "groupbyColumns": [],
+ "groupbyRows": [],
+ "metrics": [
+ {
+ "aggregate": "COUNT",
+ "column": {
+ "column_name": "is_software_dev",
+ "description": None,
+ "expression": None,
+ "filterable": True,
+ "groupby": True,
+ "id": 1463,
+ "is_dttm": False,
+ "python_date_format": None,
+ "type": "DOUBLE PRECISION",
+ "verbose_name": None,
+ },
+ "expressionType": "SIMPLE",
+ "hasCustomLabel": False,
+ "isNew": False,
+ "label": "COUNT(is_software_dev)",
+ "optionName": "metric_9i1kctig9yr_sizo6ihd2o",
+ "sqlExpression": None,
+ }
+ ],
+ "metricsLayout": "COLUMNS",
+ "adhoc_filters": [
+ {
+ "clause": "WHERE",
+ "comparator": "Currently A Developer",
+ "expressionType": "SIMPLE",
+ "filterOptionName": "filter_fvi0jg9aii_2lekqrhy7qk",
+ "isExtra": False,
+ "isNew": False,
+ "operator": "==",
+ "sqlExpression": None,
+ "subject": "developer_type",
+ }
+ ],
+ "row_limit": 10000,
+ "order_desc": True,
+ "aggregateFunction": "Sum",
+ "valueFormat": "SMART_NUMBER",
+ "date_format": "smart_date",
+ "rowOrder": "key_a_to_z",
+ "colOrder": "key_a_to_z",
+ "extra_form_data": {},
+ "force": False,
+ "result_format": "json",
+ "result_type": "results",
+ }
+
+ assert apply_client_processing(result, form_data) == {
+ "queries": [
+ {
+ "result_format": ChartDataResultFormat.CSV,
+ "data": "COUNT(is_software_dev)\n4725\n",
+ "colnames": ["COUNT(is_software_dev)"],
+ "indexnames": [0],
+ "coltypes": [GenericDataType.NUMERIC],
+ "rowcount": 1,
+ }
+ ]
+ }
+
+
def test_apply_client_processing_csv_format_empty_string():
"""
It should be able to process csv results with no data
diff --git a/tests/unit_tests/utils/csv_tests.py b/tests/unit_tests/utils/csv_tests.py
index fa2c79eb88..1ba03d2029 100644
--- a/tests/unit_tests/utils/csv_tests.py
+++ b/tests/unit_tests/utils/csv_tests.py
@@ -15,19 +15,27 @@
# specific language governing permissions and limitations
# under the License.
+import json
+
import pandas as pd
import pyarrow as pa
import pytest # noqa: F401
+from pandas.api.types import is_datetime64_any_dtype
from superset.utils import csv
+from superset.utils.core import GenericDataType
+from superset.utils.csv import (
+ df_to_escaped_csv,
+ get_chart_dataframe,
+)
def test_escape_value():
result = csv.escape_value("value")
assert result == "value"
- result = csv.escape_value("-10")
- assert result == "-10"
+ result = csv.escape_value("10")
+ assert result == "10"
result = csv.escape_value("@value")
assert result == "'@value"
@@ -35,8 +43,8 @@ def test_escape_value():
result = csv.escape_value("+value")
assert result == "'+value"
- result = csv.escape_value("-value")
- assert result == "'-value"
+ result = csv.escape_value("value")
+ assert result == "'value"
result = csv.escape_value("=value")
assert result == "'=value"
@@ -57,6 +65,84 @@ def test_escape_value():
assert result == "' =10+2"
+def fake_get_chart_csv_data_none(chart_url, auth_cookies=None):
+ return None
+
+
+def fake_get_chart_csv_data_empty(chart_url, auth_cookies=None):
+ # Return JSON with empty data so that the resulting DataFrame is empty
+ fake_result = {
+ "result": [{"data": {}, "coltypes": [], "colnames": [], "indexnames": []}]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_valid(chart_url, auth_cookies=None):
+ # Return JSON with non-temporal data and valid indexnames so that they are used.
+ fake_result = {
+ "result": [
+ {
+ "data": {"col1": [1, 2], "col2": ["a", "b"]},
+ "coltypes": [GenericDataType.NUMERIC, GenericDataType.STRING],
+ "colnames": ["col1", "col2"],
+ # Provide two index names so that a MultiIndex is built.
+ "indexnames": ["idx1", "idx2"],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_temporal(chart_url, auth_cookies=None):
+ """
+ Return JSON with a temporal column and valid indexnames
+ so that a MultiIndex is built.
+ """
+ fake_result = {
+ "result": [
+ {
+ "data": {"date": [1609459200000, 1612137600000], "val": [10, 20]},
+ "coltypes": [GenericDataType.TEMPORAL, GenericDataType.NUMERIC],
+ "colnames": ["date", "val"],
+ # Provide two index names so a MultiIndex is built.
+ "indexnames": [0, 1],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_hierarchical(chart_url, auth_cookies=None):
+ # Return JSON with hierarchical column (list-based) and matching index names.
+ fake_result = {
+ "result": [
+ {
+ "data": {"a": [1, 2]},
+ "coltypes": [GenericDataType.NUMERIC],
+ "colnames": [["level1", "a"]],
+ # Provide two index tuples for two rows
+ "indexnames": [["idx"], ["idx"]],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_default(chart_url, auth_cookies=None):
+ # When indexnames is empty, pandas should fall back to a default RangeIndex
+ fake_result = {
+ "result": [
+ {
+ "data": {"col1": [1, 2], "col2": ["a", "b"]},
+ "coltypes": [GenericDataType.NUMERIC, GenericDataType.STRING],
+ "colnames": ["col1", "col2"],
+ "indexnames": None,
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
def test_df_to_escaped_csv():
df = pd.DataFrame(
data={
@@ -73,7 +159,7 @@ def test_df_to_escaped_csv():
}
)
- escaped_csv_str = csv.df_to_escaped_csv(
+ escaped_csv_str = df_to_escaped_csv(
df,
encoding="utf8",
index=False,
@@ -94,4 +180,114 @@ def test_df_to_escaped_csv():
]
df = pa.array([1, None]).to_pandas(integer_object_nulls=True).to_frame()
- assert csv.df_to_escaped_csv(df, encoding="utf8", index=False) == '0\n1\n""\n'
+ assert df_to_escaped_csv(df, encoding="utf8", index=False) == '0\n1\n""\n'
+
+
+def test_get_chart_dataframe_returns_none_when_no_content(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_none)
+ result = get_chart_dataframe("http://dummy-url")
+ assert result is None
+
+
+def test_get_chart_dataframe_returns_none_for_empty_data(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_empty)
+ result = get_chart_dataframe("http://dummy-url")
+ # When data is empty, the function should return None
+ assert result is None
+
+
+def test_get_chart_dataframe_valid_non_temporal(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_valid)
+ df = get_chart_dataframe("http://dummy-url")
+ assert df is not None
+
+ expected_columns = pd.MultiIndex.from_tuples([("col1",), ("col2",)])
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([("idx1",), ("idx2",)])
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ pd.testing.assert_series_equal(
+ df[("col1",)], pd.Series([1, 2], name=("col1",), index=df.index)
+ )
+ pd.testing.assert_series_equal(
+ df[("col2",)], pd.Series(["a", "b"], name=("col2",), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('col1',) | ('col2',) |
+|:----------|------------:|:------------|
+| ('idx1',) | 1 | a |
+| ('idx2',) | 2 | b |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()
+
+
+def test_get_chart_dataframe_valid_temporal(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_temporal)
+ df = get_chart_dataframe("http://dummy-url")
+ expected_columns = pd.MultiIndex.from_tuples([("date",), ("val",)])
+ assert df is not None
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([(0,), (1,)])
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ assert is_datetime64_any_dtype(df[("date",)])
+ expected_dates = pd.to_datetime([1609459200000, 1612137600000], unit="ms").astype(
+ "datetime64[ms]"
+ )
+ actual_dates = df[("date",)].reset_index(drop=True)
+ pd.testing.assert_series_equal(
+ actual_dates, pd.Series(expected_dates, name=("date",)), check_names=False
+ )
+ pd.testing.assert_series_equal(
+ df[("val",)], pd.Series([10, 20], name=("val",), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('date',) | ('val',) |
+|:-----|:--------------------|-----------:|
+| (0,) | 2021-01-01 00:00:00 | 10 |
+| (1,) | 2021-02-01 00:00:00 | 20 |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()
+
+
+def test_get_chart_dataframe_with_hierarchical_columns(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_hierarchical)
+ df = get_chart_dataframe("http://dummy-url")
+ assert df is not None
+ expected_columns = pd.MultiIndex.from_tuples([("level1", "a")])
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([("idx",)] * len(df))
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ pd.testing.assert_series_equal(
+ df[("level1", "a")], pd.Series([1, 2], name=("level1", "a"), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('level1', 'a') |
+|:---------|------------------:|
+| ('idx',) | 1 |
+| ('idx',) | 2 |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()
+
+
+def test_get_chart_dataframe_default_range_index(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_default)
+ df = get_chart_dataframe("http://dummy-url")
+ assert df is not None
+ expected_columns = pd.MultiIndex.from_tuples([("col1",), ("col2",)])
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+ # When indexnames is empty, the DataFrame should have a default RangeIndex
+ assert isinstance(df.index, pd.RangeIndex)
+ pd.testing.assert_series_equal(df[("col1",)], pd.Series([1, 2], name=("col1",)))
+ pd.testing.assert_series_equal(df[("col2",)], pd.Series(["a", "b"], name=("col2",)))