You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2025/03/14 22:21:35 UTC
(superset) branch elizabeth/csv-tests updated (8d6a637bff -> e3e21e88a7)
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a change to branch elizabeth/csv-tests
in repository https://gitbox.apache.org/repos/asf/superset.git
omit 8d6a637bff add more csv tests
new e3e21e88a7 add more csv tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8d6a637bff)
\
N -- N -- N refs/heads/elizabeth/csv-tests (e3e21e88a7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tests/unit_tests/utils/csv_tests.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
(superset) 01/01: add more csv tests
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch elizabeth/csv-tests
in repository https://gitbox.apache.org/repos/asf/superset.git
commit e3e21e88a7cca706afc96333bf0e4a8650cf6abf
Author: Elizabeth Thompson <es...@gmail.com>
AuthorDate: Thu Mar 13 17:09:27 2025 -0700
add more csv tests
---
tests/unit_tests/charts/test_client_processing.py | 91 +++++++++++
tests/unit_tests/utils/csv_tests.py | 175 +++++++++++++++++++++-
2 files changed, 263 insertions(+), 3 deletions(-)
diff --git a/tests/unit_tests/charts/test_client_processing.py b/tests/unit_tests/charts/test_client_processing.py
index b38513a2e3..cecc43c45b 100644
--- a/tests/unit_tests/charts/test_client_processing.py
+++ b/tests/unit_tests/charts/test_client_processing.py
@@ -2060,6 +2060,97 @@ COUNT(is_software_dev)
}
+def test_apply_client_processing_csv_format_simple_table():
+ """
+ It should be able to process csv results
+ And not show a default column
+ """
+
+ result = {
+ "queries": [
+ {
+ "result_format": ChartDataResultFormat.CSV,
+ "data": """
+COUNT(is_software_dev)
+4725
+""",
+ }
+ ]
+ }
+ form_data = {
+ "datasource": "19__table",
+ "viz_type": "table",
+ "slice_id": 69,
+ "url_params": {},
+ "granularity_sqla": "time_start",
+ "time_grain_sqla": "P1D",
+ "time_range": "No filter",
+ "groupbyColumns": [],
+ "groupbyRows": [],
+ "metrics": [
+ {
+ "aggregate": "COUNT",
+ "column": {
+ "column_name": "is_software_dev",
+ "description": None,
+ "expression": None,
+ "filterable": True,
+ "groupby": True,
+ "id": 1463,
+ "is_dttm": False,
+ "python_date_format": None,
+ "type": "DOUBLE PRECISION",
+ "verbose_name": None,
+ },
+ "expressionType": "SIMPLE",
+ "hasCustomLabel": False,
+ "isNew": False,
+ "label": "COUNT(is_software_dev)",
+ "optionName": "metric_9i1kctig9yr_sizo6ihd2o",
+ "sqlExpression": None,
+ }
+ ],
+ "metricsLayout": "COLUMNS",
+ "adhoc_filters": [
+ {
+ "clause": "WHERE",
+ "comparator": "Currently A Developer",
+ "expressionType": "SIMPLE",
+ "filterOptionName": "filter_fvi0jg9aii_2lekqrhy7qk",
+ "isExtra": False,
+ "isNew": False,
+ "operator": "==",
+ "sqlExpression": None,
+ "subject": "developer_type",
+ }
+ ],
+ "row_limit": 10000,
+ "order_desc": True,
+ "aggregateFunction": "Sum",
+ "valueFormat": "SMART_NUMBER",
+ "date_format": "smart_date",
+ "rowOrder": "key_a_to_z",
+ "colOrder": "key_a_to_z",
+ "extra_form_data": {},
+ "force": False,
+ "result_format": "json",
+ "result_type": "results",
+ }
+
+ assert apply_client_processing(result, form_data) == {
+ "queries": [
+ {
+ "result_format": ChartDataResultFormat.CSV,
+ "data": "COUNT(is_software_dev)\n4725\n",
+ "colnames": ["COUNT(is_software_dev)"],
+ "indexnames": [0],
+ "coltypes": [GenericDataType.NUMERIC],
+ "rowcount": 1,
+ }
+ ]
+ }
+
+
def test_apply_client_processing_csv_format_empty_string():
"""
It should be able to process csv results with no data
diff --git a/tests/unit_tests/utils/csv_tests.py b/tests/unit_tests/utils/csv_tests.py
index fa2c79eb88..179d241a97 100644
--- a/tests/unit_tests/utils/csv_tests.py
+++ b/tests/unit_tests/utils/csv_tests.py
@@ -15,11 +15,19 @@
# specific language governing permissions and limitations
# under the License.
+import json
+
import pandas as pd
import pyarrow as pa
import pytest # noqa: F401
+from pandas.api.types import is_datetime64_any_dtype
from superset.utils import csv
+from superset.utils.core import GenericDataType
+from superset.utils.csv import (
+ df_to_escaped_csv,
+ get_chart_dataframe,
+)
def test_escape_value():
@@ -36,7 +44,7 @@ def test_escape_value():
assert result == "'+value"
result = csv.escape_value("-value")
- assert result == "'-value"
+ assert result == "-value"
result = csv.escape_value("=value")
assert result == "'=value"
@@ -57,6 +65,69 @@ def test_escape_value():
assert result == "' =10+2"
+def fake_get_chart_csv_data_none(chart_url, auth_cookies=None):
+ return None
+
+
+def fake_get_chart_csv_data_empty(chart_url, auth_cookies=None):
+ # Return JSON with empty data so that the resulting DataFrame is empty
+ fake_result = {
+ "result": [{"data": {}, "coltypes": [], "colnames": [], "indexnames": []}]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_valid(chart_url, auth_cookies=None):
+ # Return JSON with non-temporal data and valid indexnames so that they are used.
+ fake_result = {
+ "result": [
+ {
+ "data": {"col1": [1, 2], "col2": ["a", "b"]},
+ "coltypes": [GenericDataType.NUMERIC, GenericDataType.STRING],
+ "colnames": ["col1", "col2"],
+ # Provide two index names so that a MultiIndex is built.
+ "indexnames": ["idx1", "idx2"],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_temporal(chart_url, auth_cookies=None):
+ """
+ Return JSON with a temporal column and valid indexnames
+ so that a MultiIndex is built.
+ """
+ fake_result = {
+ "result": [
+ {
+ "data": {"date": [1609459200000, 1612137600000], "val": [10, 20]},
+ "coltypes": [GenericDataType.TEMPORAL, GenericDataType.NUMERIC],
+ "colnames": ["date", "val"],
+ # Provide two index names so a MultiIndex is built.
+ "indexnames": [0, 1],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
+def fake_get_chart_csv_data_hierarchical(chart_url, auth_cookies=None):
+ # Return JSON with hierarchical column (list-based) and matching index names.
+ fake_result = {
+ "result": [
+ {
+ "data": {"a": [1, 2]},
+ "coltypes": [GenericDataType.NUMERIC],
+ "colnames": [["level1", "a"]],
+ # Provide two index tuples for two rows
+ "indexnames": [["idx"], ["idx"]],
+ }
+ ]
+ }
+ return json.dumps(fake_result).encode("utf-8")
+
+
def test_df_to_escaped_csv():
df = pd.DataFrame(
data={
@@ -73,7 +144,7 @@ def test_df_to_escaped_csv():
}
)
- escaped_csv_str = csv.df_to_escaped_csv(
+ escaped_csv_str = df_to_escaped_csv(
df,
encoding="utf8",
index=False,
@@ -94,4 +165,102 @@ def test_df_to_escaped_csv():
]
df = pa.array([1, None]).to_pandas(integer_object_nulls=True).to_frame()
- assert csv.df_to_escaped_csv(df, encoding="utf8", index=False) == '0\n1\n""\n'
+ assert df_to_escaped_csv(df, encoding="utf8", index=False) == '0\n1\n""\n'
+
+
+def test_get_chart_dataframe_returns_none_when_no_content(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_none)
+ result = get_chart_dataframe("http://dummy-url")
+ assert result is None
+
+
+def test_get_chart_dataframe_returns_none_for_empty_data(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_empty)
+ result = get_chart_dataframe("http://dummy-url")
+ # When data is empty, the function should return None
+ assert result is None
+
+
+def test_get_chart_dataframe_valid_non_temporal(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_valid)
+ df = get_chart_dataframe("http://dummy-url")
+ assert df is not None
+
+ expected_columns = pd.MultiIndex.from_tuples([("col1",), ("col2",)])
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([("idx1",), ("idx2",)])
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ pd.testing.assert_series_equal(
+ df[("col1",)], pd.Series([1, 2], name=("col1",), index=df.index)
+ )
+ pd.testing.assert_series_equal(
+ df[("col2",)], pd.Series(["a", "b"], name=("col2",), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('col1',) | ('col2',) |
+|:----------|------------:|:------------|
+| ('idx1',) | 1 | a |
+| ('idx2',) | 2 | b |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()
+
+
+def test_get_chart_dataframe_valid_temporal(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_temporal)
+ df = get_chart_dataframe("http://dummy-url")
+ expected_columns = pd.MultiIndex.from_tuples([("date",), ("val",)])
+ assert df is not None
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([(0,), (1,)])
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ assert is_datetime64_any_dtype(df[("date",)])
+ expected_dates = pd.to_datetime([1609459200000, 1612137600000], unit="ms").astype(
+ "datetime64[ms]"
+ )
+ actual_dates = df[("date",)].reset_index(drop=True)
+ pd.testing.assert_series_equal(
+ actual_dates, pd.Series(expected_dates, name=("date",)), check_names=False
+ )
+ pd.testing.assert_series_equal(
+ df[("val",)], pd.Series([10, 20], name=("val",), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('date',) | ('val',) |
+|:-----|:--------------------|-----------:|
+| (0,) | 2021-01-01 00:00:00 | 10 |
+| (1,) | 2021-02-01 00:00:00 | 20 |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()
+
+
+def test_get_chart_dataframe_with_hierarchical_columns(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(csv, "get_chart_csv_data", fake_get_chart_csv_data_hierarchical)
+ df = get_chart_dataframe("http://dummy-url")
+ assert df is not None
+ expected_columns = pd.MultiIndex.from_tuples([("level1", "a")])
+ pd.testing.assert_index_equal(df.columns, expected_columns)
+
+ expected_index = pd.MultiIndex.from_tuples([("idx",)] * len(df))
+ pd.testing.assert_index_equal(df.index, expected_index)
+
+ pd.testing.assert_series_equal(
+ df[("level1", "a")], pd.Series([1, 2], name=("level1", "a"), index=df.index)
+ )
+ markdown_str = df.to_markdown()
+ expected_markdown_str = """
+| | ('level1', 'a') |
+|:---------|------------------:|
+| ('idx',) | 1 |
+| ('idx',) | 2 |
+"""
+ assert markdown_str.strip() == expected_markdown_str.strip()