You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2025/09/09 13:13:29 UTC
(superset) branch 5.0-extensions updated (e7296ea709 -> bf7c97d807)
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a change to branch 5.0-extensions
in repository https://gitbox.apache.org/repos/asf/superset.git
from e7296ea709 fix: apache-superset-extensions-cli exported files
new fea386309a fix(echarts): rename time series shifted for isTimeComparisonValue (#35022)
new bf7c97d807 fix: Upload CSV as Dataset (#34763)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/operators/renameOperator.ts | 14 +-
.../test/operators/renameOperator.test.ts | 38 ++
superset/commands/database/uploaders/base.py | 3 +-
superset/commands/database/uploaders/csv_reader.py | 229 ++++++++-
superset/models/core.py | 4 +-
.../commands/databases/csv_reader_test.py | 556 +++++++++++++++++++++
6 files changed, 822 insertions(+), 22 deletions(-)
(superset) 02/02: fix: Upload CSV as Dataset (#34763)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 5.0-extensions
in repository https://gitbox.apache.org/repos/asf/superset.git
commit bf7c97d807958f3334fcd80432c7fb8d29c88f2c
Author: Luiz Otavio <45...@users.noreply.github.com>
AuthorDate: Mon Sep 8 11:48:23 2025 -0300
fix: Upload CSV as Dataset (#34763)
(cherry picked from commit 1c2b9db4f0d5728f9bcf35ebdef2df5c41d35590)
---
superset/commands/database/uploaders/base.py | 3 +-
superset/commands/database/uploaders/csv_reader.py | 229 ++++++++-
superset/models/core.py | 4 +-
.../commands/databases/csv_reader_test.py | 556 +++++++++++++++++++++
4 files changed, 776 insertions(+), 16 deletions(-)
diff --git a/superset/commands/database/uploaders/base.py b/superset/commands/database/uploaders/base.py
index 5a8227b054..68a437a203 100644
--- a/superset/commands/database/uploaders/base.py
+++ b/superset/commands/database/uploaders/base.py
@@ -133,7 +133,8 @@ class BaseDataReader:
)
) from ex
except Exception as ex:
- raise DatabaseUploadFailed(exception=ex) from ex
+ message = ex.message if hasattr(ex, "message") and ex.message else str(ex)
+ raise DatabaseUploadFailed(message=message, exception=ex) from ex
class UploadCommand(BaseCommand):
diff --git a/superset/commands/database/uploaders/csv_reader.py b/superset/commands/database/uploaders/csv_reader.py
index 0072a7d5cf..cc1102e26b 100644
--- a/superset/commands/database/uploaders/csv_reader.py
+++ b/superset/commands/database/uploaders/csv_reader.py
@@ -33,6 +33,10 @@ logger = logging.getLogger(__name__)
READ_CSV_CHUNK_SIZE = 1000
ROWS_TO_READ_METADATA = 2
+# Fixed error limit to avoid huge payloads and poor UX given that a file
+# might contain thousands of errors.
+MAX_DISPLAYED_ERRORS = 5
+
class CSVReaderOptions(ReaderOptions, total=False):
delimiter: str
@@ -60,20 +64,211 @@ class CSVReader(BaseDataReader):
options=dict(options),
)
+ @staticmethod
+ def _find_invalid_values_numeric(df: pd.DataFrame, column: str) -> pd.Series:
+ """
+ Find invalid values for numeric type conversion.
+ Identifies rows where values cannot be converted to numeric types using
+ pandas to_numeric with error coercing. Returns a boolean mask indicating
+ which values are invalid (non-null but unconvertible).
+ :param df: DataFrame containing the data
+ :param column: Name of the column to check for invalid values
+ :return: Boolean Series indicating which rows have invalid
+ values for numeric conversion
+ """
+ converted = pd.to_numeric(df[column], errors="coerce")
+ return converted.isna() & df[column].notna()
+
+ @staticmethod
+ def _find_invalid_values_non_numeric(
+ df: pd.DataFrame, column: str, dtype: str
+ ) -> pd.Series:
+ """
+ Find invalid values for non-numeric type conversion.
+ Identifies rows where values cannot be converted to the specified non-numeric
+ data type by attempting conversion and catching exceptions. This is used for
+ string, categorical, or other non-numeric type conversions.
+ :param df: DataFrame containing the data
+ :param column: Name of the column to check for invalid values
+ :param dtype: Target data type for conversion (e.g., 'string', 'category')
+ :return: Boolean Series indicating which rows have
+ invalid values for the target type
+ """
+ invalid_mask = pd.Series([False] * len(df), index=df.index)
+ for idx, value in df[column].items():
+ if pd.notna(value):
+ try:
+ pd.Series([value]).astype(dtype)
+ except (ValueError, TypeError):
+ invalid_mask[idx] = True
+ return invalid_mask
+
+ @staticmethod
+ def _get_error_details(
+ df: pd.DataFrame,
+ column: str,
+ dtype: str,
+ invalid_mask: pd.Series,
+ kwargs: dict[str, Any],
+ ) -> tuple[list[str], int]:
+ """
+ Get detailed error information for invalid values in type conversion.
+ Extracts detailed information about conversion errors, including specific
+ invalid values and their line numbers. Limits the number of detailed errors
+ shown to avoid overwhelming output while providing total error count.
+ :param df: DataFrame containing the data
+ :param column: Name of the column with conversion errors
+ :param dtype: Target data type that failed conversion
+ :param invalid_mask: Boolean mask indicating which rows have invalid values
+ :param kwargs: Additional parameters including header row information
+ :return: Tuple containing:
+ - List of formatted error detail strings (limited by MAX_DISPLAYED_ERRORS)
+ - Total count of errors found
+ """
+ if not invalid_mask.any():
+ return [], 0
+
+ invalid_indices = invalid_mask[invalid_mask].index.tolist()
+ total_errors = len(invalid_indices)
+
+ error_details = []
+ for idx in invalid_indices[:MAX_DISPLAYED_ERRORS]:
+ invalid_value = df.loc[idx, column]
+ line_number = idx + kwargs.get("header", 0) + 2
+ error_details.append(
+ f" • Line {line_number}: '{invalid_value}' cannot be converted to "
+ f"{dtype}"
+ )
+
+ return error_details, total_errors
+
+ @staticmethod
+ def _create_error_message(
+ df: pd.DataFrame,
+ column: str,
+ dtype: str,
+ invalid_mask: pd.Series,
+ kwargs: dict[str, Any],
+ original_error: Exception,
+ ) -> str:
+ """
+ Create detailed error message for type conversion failure.
+ Constructs a comprehensive error message that includes:
+ - Column name and target type
+ - Total count of errors found
+ - Detailed list of first few errors with line numbers and values
+ - Summary of remaining errors if exceeding display limit
+ :param df: DataFrame containing the data
+ :param column: Name of the column that failed conversion
+ :param dtype: Target data type that failed
+ :param invalid_mask: Boolean mask indicating which rows have invalid values
+ :param kwargs: Additional parameters including header information
+ :param original_error: Original exception that triggered the error handling
+ :return: Formatted error message string ready for display to user
+ """
+ error_details, total_errors = CSVReader._get_error_details(
+ df, column, dtype, invalid_mask, kwargs
+ )
+
+ if error_details:
+ base_msg = (
+ f"Cannot convert column '{column}' to {dtype}. "
+ f"Found {total_errors} error(s):"
+ )
+ detailed_errors = "\n".join(error_details)
+
+ if total_errors > MAX_DISPLAYED_ERRORS:
+ remaining = total_errors - MAX_DISPLAYED_ERRORS
+ additional_msg = f"\n ... and {remaining} more error(s)"
+ return f"{base_msg}\n{detailed_errors}{additional_msg}"
+ else:
+ return f"{base_msg}\n{detailed_errors}"
+ else:
+ return f"Cannot convert column '{column}' to {dtype}. {str(original_error)}"
+
+ @staticmethod
+ def _cast_single_column(
+ df: pd.DataFrame, column: str, dtype: str, kwargs: dict[str, Any]
+ ) -> None:
+ """
+ Cast a single DataFrame column to the specified data type.
+ Attempts to convert a column to the target data type with enhanced error
+ handling. For numeric types, uses pandas to_numeric for better performance
+ and error detection. If conversion fails, provides detailed
+ error messages including specific invalid values and their line numbers.
+ :param df: DataFrame to modify (modified in-place)
+ :param column: Name of the column to cast
+ :param dtype: Target data type (e.g., 'int64', 'float64', 'string')
+ :param kwargs: Additional parameters including header row information
+ :raises DatabaseUploadFailed: If type conversion fails,
+ with detailed error message
+ """
+ numeric_types = {"int64", "int32", "float64", "float32"}
+
+ try:
+ if dtype in numeric_types:
+ df[column] = pd.to_numeric(df[column], errors="raise")
+ df[column] = df[column].astype(dtype)
+ else:
+ df[column] = df[column].astype(dtype)
+ except (ValueError, TypeError) as ex:
+ try:
+ if dtype in numeric_types:
+ invalid_mask = CSVReader._find_invalid_values_numeric(df, column)
+ else:
+ invalid_mask = CSVReader._find_invalid_values_non_numeric(
+ df, column, dtype
+ )
+
+ error_msg = CSVReader._create_error_message(
+ df, column, dtype, invalid_mask, kwargs, ex
+ )
+ except Exception:
+ error_msg = f"Cannot convert column '{column}' to {dtype}. {str(ex)}"
+
+ raise DatabaseUploadFailed(message=error_msg) from ex
+
+ @staticmethod
+ def _cast_column_types(
+ df: pd.DataFrame, types: dict[str, str], kwargs: dict[str, Any]
+ ) -> pd.DataFrame:
+ """
+ Cast DataFrame columns to specified types with detailed
+ error reporting.
+ :param df: DataFrame to cast
+ :param types: Dictionary mapping column names to target types
+ :param kwargs: Original read_csv kwargs for line number calculation
+ :return: DataFrame with casted columns
+ :raises DatabaseUploadFailed: If type conversion fails with detailed error info
+ """
+ for column, dtype in types.items():
+ if column not in df.columns:
+ continue
+ CSVReader._cast_single_column(df, column, dtype, kwargs)
+ return df
+
@staticmethod
def _read_csv(file: FileStorage, kwargs: dict[str, Any]) -> pd.DataFrame:
try:
+ types = kwargs.pop("dtype", None)
+ df = None
if "chunksize" in kwargs:
- return pd.concat(
+ df = pd.concat(
pd.read_csv(
filepath_or_buffer=file.stream,
**kwargs,
)
)
- return pd.read_csv(
- filepath_or_buffer=file.stream,
- **kwargs,
- )
+ else:
+ df = pd.read_csv(
+ filepath_or_buffer=file.stream,
+ **kwargs,
+ )
+ if types:
+ df = CSVReader._cast_column_types(df, types, kwargs)
+ return df
+ except DatabaseUploadFailed:
+ raise
except (
pd.errors.ParserError,
pd.errors.EmptyDataError,
@@ -102,21 +297,27 @@ class CSVReader(BaseDataReader):
"dayfirst": self._options.get("day_first", False),
"iterator": True,
"keep_default_na": not self._options.get("null_values"),
- "usecols": self._options.get("columns_read")
- if self._options.get("columns_read") # None if an empty list
- else None,
- "na_values": self._options.get("null_values")
- if self._options.get("null_values") # None if an empty list
- else None,
+ "usecols": (
+ self._options.get("columns_read")
+ if self._options.get("columns_read") # None if an empty list
+ else None
+ ),
+ "na_values": (
+ self._options.get("null_values")
+ if self._options.get("null_values") # None if an empty list
+ else None
+ ),
"nrows": self._options.get("rows_to_read"),
"parse_dates": self._options.get("column_dates"),
"sep": self._options.get("delimiter", ","),
"skip_blank_lines": self._options.get("skip_blank_lines", False),
"skipinitialspace": self._options.get("skip_initial_space", False),
"skiprows": self._options.get("skip_rows", 0),
- "dtype": self._options.get("column_data_types")
- if self._options.get("column_data_types")
- else None,
+ "dtype": (
+ self._options.get("column_data_types")
+ if self._options.get("column_data_types")
+ else None
+ ),
}
return self._read_csv(file, kwargs)
diff --git a/superset/models/core.py b/superset/models/core.py
index 4db7b04d56..361d0c37a3 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -1094,7 +1094,9 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
def has_table(self, table: Table) -> bool:
with self.get_sqla_engine(catalog=table.catalog, schema=table.schema) as engine:
# do not pass "" as an empty schema; force null
- return engine.has_table(table.table, table.schema or None)
+ if engine.has_table(table.table, table.schema or None):
+ return True
+ return engine.has_table(table.table.lower(), table.schema or None)
def has_view(self, table: Table) -> bool:
with self.get_sqla_engine(catalog=table.catalog, schema=table.schema) as engine:
diff --git a/tests/unit_tests/commands/databases/csv_reader_test.py b/tests/unit_tests/commands/databases/csv_reader_test.py
index 2fb6bc5c6c..a992eac5dc 100644
--- a/tests/unit_tests/commands/databases/csv_reader_test.py
+++ b/tests/unit_tests/commands/databases/csv_reader_test.py
@@ -18,6 +18,7 @@ import io
from datetime import datetime
import numpy as np
+import pandas as pd
import pytest
from werkzeug.datastructures import FileStorage
@@ -371,3 +372,558 @@ def test_csv_reader_file_metadata_invalid_file():
"Parsing error: Error tokenizing data. C error:"
" Expected 3 fields in line 3, saw 7\n"
)
+
+
+def test_csv_reader_integer_in_float_column():
+ csv_data = [
+ ["Name", "Score", "City"],
+ ["name1", 25.5, "city1"],
+ ["name2", 25, "city2"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Score": "float"})
+ )
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (2, 3)
+ assert df["Score"].dtype == "float64"
+
+
+def test_csv_reader_object_type_auto_inferring():
+ # this case below won't raise a error
+ csv_data = [
+ ["Name", "id", "City"],
+ ["name1", 25.5, "city1"],
+ ["name2", 15, "city2"],
+ ["name3", 123456789086, "city3"],
+ ["name4", "abc", "city4"],
+ ["name5", 4.75, "city5"],
+ ]
+
+ csv_reader = CSVReader()
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (5, 3)
+ # pandas automatically infers the type if column_data_types is not informed
+ # if there's only one string in the column it converts the whole column to object
+ assert df["id"].dtype == "object"
+
+
+def test_csv_reader_float_type_auto_inferring():
+ csv_data = [
+ ["Name", "id", "City"],
+ ["name1", "25", "city1"],
+ ["name2", "15", "city2"],
+ ["name3", "123456789086", "city3"],
+ ["name5", "4.75", "city5"],
+ ]
+
+ csv_reader = CSVReader()
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (4, 3)
+ # The type here is automatically inferred to float due to 4.75 value
+ assert df["id"].dtype == "float64"
+
+
+def test_csv_reader_int_type_auto_inferring():
+ csv_data = [
+ ["Name", "id", "City"],
+ ["name1", "0", "city1"],
+ ["name2", "15", "city2"],
+ ["name3", "123456789086", "city3"],
+ ["name5", "45", "city5"],
+ ]
+
+ csv_reader = CSVReader()
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (4, 3)
+ assert df["id"].dtype == "int64"
+
+
+def test_csv_reader_bigint_type_auto_inferring():
+ csv_data = [
+ ["Name", "id", "City"],
+ ["name1", "9223372036854775807", "city1"],
+ ["name2", "9223372036854775806", "city2"],
+ ["name3", "1234567890123456789", "city3"],
+ ["name4", "0", "city4"],
+ ["name5", "-9223372036854775808", "city5"],
+ ]
+
+ csv_reader = CSVReader()
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (5, 3)
+ assert df["id"].dtype == "int64"
+ assert df.iloc[0]["id"] == 9223372036854775807
+ assert df.iloc[4]["id"] == -9223372036854775808
+
+
+def test_csv_reader_int_typing():
+ csv_data = [
+ ["Name", "id", "City"],
+ ["name1", "0", "city1"],
+ ["name2", "15", "city2"],
+ ["name3", "123456789086", "city3"],
+ ["name5", "45", "city5"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"id": "int"}))
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (4, 3)
+ assert df["id"].dtype == "int64"
+
+
+def test_csv_reader_float_typing():
+ csv_data = [
+ ["Name", "score", "City"],
+ ["name1", "0", "city1"],
+ ["name2", "15.3", "city2"],
+ ["name3", "45", "city3"],
+ ["name5", "23.1342", "city5"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"score": "float"})
+ )
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (4, 3)
+ assert df["score"].dtype == "float64"
+
+
+def test_csv_reader_multiple_errors_display():
+ """Test that multiple errors are displayed with proper formatting."""
+ csv_data = [
+ ["Name", "Age", "Score"],
+ ["Alice", "25", "95.5"],
+ ["Bob", "invalid1", "87.2"],
+ ["Charlie", "invalid2", "92.1"],
+ ["Diana", "invalid3", "88.5"],
+ ["Eve", "invalid4", "90.0"],
+ ["Frank", "30", "85.5"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 4 error(s):" in error_msg
+ assert "Line 3: 'invalid1' cannot be converted to int64" in error_msg
+ assert "Line 4: 'invalid2' cannot be converted to int64" in error_msg
+ assert "Line 5: 'invalid3' cannot be converted to int64" in error_msg
+ assert "Line 6: 'invalid4' cannot be converted to int64" in error_msg
+ # With MAX_DISPLAYED_ERRORS = 5, all 4 errors should be shown without truncation
+ assert "and" not in error_msg or "more error(s)" not in error_msg
+
+
+def test_csv_reader_non_numeric_in_integer_column():
+ csv_data = [
+ ["Name", "Age", "City"],
+ ["name1", "abc", "city1"],
+ ["name2", "25", "city2"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 2: 'abc' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_non_numeric_in_float_column():
+ csv_data = [
+ ["Name", "Score", "City"],
+ ["name1", "5.3", "city1"],
+ ["name2", "25.5", "city2"],
+ ["name3", "24.5", "city3"],
+ ["name4", "1.0", "city4"],
+ ["name5", "one point five", "city5"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Score": "float64"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Score' to float64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 6: 'one point five' cannot be converted to float64" in error_msg
+
+
+def test_csv_reader_improved_error_detection_int32():
+ """Test improved error detection for int32 type casting."""
+ csv_data = [
+ ["Name", "ID", "City"],
+ ["name1", "123", "city1"],
+ ["name2", "456", "city2"],
+ ["name3", "not_a_number", "city3"],
+ ["name4", "789", "city4"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"ID": "int32"}))
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'ID' to int32" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'not_a_number' cannot be converted to int32" in error_msg
+
+
+def test_csv_reader_improved_error_detection_float32():
+ """Test improved error detection for float32 type casting."""
+ csv_data = [
+ ["Name", "Score", "City"],
+ ["name1", "1.5", "city1"],
+ ["name2", "2.7", "city2"],
+ ["name3", "invalid_float", "city3"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Score": "float32"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Score' to float32" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'invalid_float' cannot be converted to float32" in error_msg
+
+
+def test_csv_reader_error_detection_with_header_row():
+ """Test that line numbers are correctly calculated with custom header row."""
+ csv_data = [
+ ["skip_this_row", "skip", "skip"],
+ ["Name", "Age", "City"],
+ ["name1", "25", "city1"],
+ ["name2", "invalid_age", "city2"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(header_row=1, column_data_types={"Age": "int"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'invalid_age' cannot be converted to int" in error_msg
+
+
+def test_csv_reader_error_detection_first_row_error():
+ """Test error detection when the first data row has the error."""
+
+ csv_data = [
+ ["Name", "Age", "City"],
+ ["name1", "not_a_number", "city1"],
+ ["name2", "25", "city2"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 2: 'not_a_number' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_error_detection_missing_column():
+ """Test that missing columns are handled gracefully."""
+ csv_data = [
+ ["Name", "City"],
+ ["name1", "city1"],
+ ["name2", "city2"],
+ ]
+
+ # Try to cast a column that doesn't exist
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"NonExistent": "int64"})
+ )
+
+ # Should not raise an error for missing columns
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+ assert df.shape == (2, 2)
+ assert df.columns.tolist() == ["Name", "City"]
+
+
+def test_csv_reader_error_detection_mixed_valid_invalid():
+ csv_data = [
+ ["Name", "Score", "City"],
+ ["name1", "95.5", "city1"],
+ ["name2", "87.2", "city2"],
+ ["name3", "92.1", "city3"],
+ ["name4", "eighty-five", "city4"],
+ ["name5", "78.9", "city5"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Score": "float64"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Score' to float64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 5: 'eighty-five' cannot be converted to float64" in error_msg
+
+
+def test_csv_reader_error_detection_multiple_invalid_values():
+ """Test error detection with multiple invalid values showing first 5 + count."""
+ csv_data = [
+ ["Name", "Score", "City"],
+ ["name1", "95.5", "city1"],
+ ["name2", "87.2", "city2"],
+ ["name3", "92.1", "city3"],
+ ["name4", "eighty-five", "city4"],
+ ["name4", "eighty-one", "city4"],
+ ["name4", "eighty", "city4"],
+ ["name4", "one", "city4"],
+ ["name4", "two", "city4"],
+ ["name4", "three", "city4"],
+ ["name5", "78.9", "city5"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Score": "float64"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Score' to float64" in error_msg
+ assert "Found 6 error(s):" in error_msg
+ assert "Line 5: 'eighty-five' cannot be converted to float64" in error_msg
+ assert "Line 6: 'eighty-one' cannot be converted to float64" in error_msg
+ assert "Line 7: 'eighty' cannot be converted to float64" in error_msg
+ assert "Line 8: 'one' cannot be converted to float64" in error_msg
+ assert "Line 9: 'two' cannot be converted to float64" in error_msg
+ assert "and 1 more error(s)" in error_msg
+
+
+def test_csv_reader_error_detection_non_numeric_types():
+ """Test error detection for non-numeric type casting."""
+ csv_data = [
+ ["Name", "Status", "City"],
+ ["name1", "active", "city1"],
+ ["name2", "inactive", "city2"],
+ ["name3", 123, "city3"], # This should cause an error when casting to string
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Status": "string"})
+ )
+
+ # For non-numeric types, the error detection should still work
+ # but might have different behavior depending on pandas version
+ try:
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+ # If no error is raised, the conversion succeeded
+ assert df["Status"].dtype == "string"
+ except DatabaseUploadFailed as ex:
+ # If an error is raised, it should have proper formatting
+ error_msg = str(ex.value)
+ assert "Cannot convert" in error_msg
+ assert "Status" in error_msg
+
+
+def test_csv_reader_error_detection_with_null_values():
+ csv_data = [
+ ["Name", "Age", "City"],
+ ["name1", "25", "city1"],
+ ["name2", "", "city2"],
+ ["name3", "invalid_age", "city3"],
+ ]
+
+ csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_successful_numeric_conversion():
+ csv_data = [
+ ["Name", "Age", "Score", "ID"],
+ ["name1", "25", "95.5", "1001"],
+ ["name2", "30", "87.2", "1002"],
+ ["name3", "35", "92.1", "1003"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(
+ column_data_types={
+ "Age": "int64",
+ "Score": "float64",
+ "ID": "int32",
+ }
+ )
+ )
+
+ df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+ assert df.shape == (3, 4)
+ assert df["Age"].dtype == "int64"
+ assert df["Score"].dtype == "float64"
+ assert df["ID"].dtype == "int32"
+ assert df.iloc[0]["Age"] == 25
+ assert df.iloc[0]["Score"] == 95.5
+ assert df.iloc[0]["ID"] == 1001
+
+
+def test_csv_reader_error_detection_improvements_summary():
+ csv_data_with_custom_header = [
+ ["metadata_row", "skip", "this"],
+ ["Name", "Age", "Score"],
+ ["Alice", "25", "95.5"],
+ ["Bob", "invalid_age", "87.2"],
+ ["Charlie", "30", "92.1"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(
+ header_row=1, column_data_types={"Age": "int64", "Score": "float64"}
+ )
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data_with_custom_header))
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+ # Test case 2: Multiple type errors - Age comes first alphabetically
+ csv_data_multiple_errors = [
+ ["Name", "Age", "Score"],
+ ["Alice", "25", "95.5"],
+ ["Bob", "invalid_age", "invalid_score"], # Error in both columns (line 3)
+ ["Charlie", "30", "92.1"],
+ ]
+
+ csv_reader = CSVReader(
+ options=CSVReaderOptions(column_data_types={"Age": "int64", "Score": "float64"})
+ )
+
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ csv_reader.file_to_dataframe(create_csv_file(csv_data_multiple_errors))
+
+ error_msg = str(ex.value)
+ # Should catch the Age error first (Age comes before Score alphabetically)
+ assert "Cannot convert column 'Age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 3: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_cast_column_types_function():
+ """Test the _cast_column_types function directly for better isolation."""
+ # Create test DataFrame
+ test_data = {
+ "name": ["Alice", "Bob", "Charlie"],
+ "age": ["25", "30", "invalid_age"],
+ "score": ["95.5", "87.2", "92.1"],
+ }
+ df = pd.DataFrame(test_data)
+
+ # Test successful casting
+ types_success = {"age": "int64", "score": "float64"}
+ kwargs = {"header": 0}
+
+ # This should work for first two rows, but we'll only test the first two
+ df_subset = df.iloc[:2].copy()
+ result_df = CSVReader._cast_column_types(df_subset, types_success, kwargs)
+
+ assert result_df["age"].dtype == "int64"
+ assert result_df["score"].dtype == "float64"
+ assert result_df.iloc[0]["age"] == 25
+ assert result_df.iloc[0]["score"] == 95.5
+
+ # Test error case
+ with pytest.raises(DatabaseUploadFailed) as ex:
+ CSVReader._cast_column_types(df, types_success, kwargs)
+
+ error_msg = str(ex.value)
+ assert "Cannot convert column 'age' to int64" in error_msg
+ assert "Found 1 error(s):" in error_msg
+ assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_cast_column_types_missing_column():
+ """Test _cast_column_types with missing columns."""
+ test_data = {
+ "name": ["Alice", "Bob"],
+ "age": ["25", "30"],
+ }
+ df = pd.DataFrame(test_data)
+
+ # Try to cast a column that doesn't exist
+ types = {"age": "int64", "nonexistent": "float64"}
+ kwargs = {"header": 0}
+
+ # Should not raise an error for missing columns
+ result_df = CSVReader._cast_column_types(df, types, kwargs)
+ assert result_df["age"].dtype == "int64"
+ assert "nonexistent" not in result_df.columns
+
+
+def test_csv_reader_cast_column_types_different_numeric_types():
+ """Test _cast_column_types with various numeric types."""
+ test_data = {
+ "int32_col": ["1", "2", "3"],
+ "int64_col": ["100", "200", "300"],
+ "float32_col": ["1.5", "2.5", "3.5"],
+ "float64_col": ["10.1", "20.2", "30.3"],
+ }
+ df = pd.DataFrame(test_data)
+
+ types = {
+ "int32_col": "int32",
+ "int64_col": "int64",
+ "float32_col": "float32",
+ "float64_col": "float64",
+ }
+ kwargs = {"header": 0}
+
+ result_df = CSVReader._cast_column_types(df, types, kwargs)
+
+ assert result_df["int32_col"].dtype == "int32"
+ assert result_df["int64_col"].dtype == "int64"
+ assert result_df["float32_col"].dtype == "float32"
+ assert result_df["float64_col"].dtype == "float64"
(superset) 01/02: fix(echarts): rename time series shifted for isTimeComparisonValue (#35022)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 5.0-extensions
in repository https://gitbox.apache.org/repos/asf/superset.git
commit fea386309a519fde2915d11e3a653416b4d3305b
Author: JUST.in DO IT <ju...@airbnb.com>
AuthorDate: Fri Sep 5 08:39:46 2025 -0700
fix(echarts): rename time series shifted for isTimeComparisonValue (#35022)
(cherry picked from commit bc54b7970a5e84b84d7e318c23d0f97a8a8a87d0)
---
.../src/operators/renameOperator.ts | 14 ++++----
.../test/operators/renameOperator.test.ts | 38 ++++++++++++++++++++++
2 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/renameOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/renameOperator.ts
index 9e1076900f..86643e2682 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/renameOperator.ts
+++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/renameOperator.ts
@@ -43,15 +43,17 @@ export const renameOperator: PostProcessingFactory<PostProcessingRename> = (
// remove or rename top level of column name(metric name) in the MultiIndex when
// 1) at least 1 metric
- // 2) dimension exist or multiple time shift metrics exist
- // 3) xAxis exist
- // 4) truncate_metric in form_data and truncate_metric is true
+ // 2) xAxis exist
+ // 3a) isTimeComparisonValue
+ // 3b-1) dimension exist or multiple time shift metrics exist
+ // 3b-2) truncate_metric in form_data and truncate_metric is true
if (
metrics.length > 0 &&
- (columns.length > 0 || timeOffsets.length > 1) &&
xAxisLabel &&
- truncate_metric !== undefined &&
- !!truncate_metric
+ (isTimeComparisonValue ||
+ ((columns.length > 0 || timeOffsets.length > 1) &&
+ truncate_metric !== undefined &&
+ !!truncate_metric))
) {
const renamePairs: [string, string | null][] = [];
if (
diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/operators/renameOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/operators/renameOperator.test.ts
index 19d594a7f6..26669a7510 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/test/operators/renameOperator.test.ts
+++ b/superset-frontend/packages/superset-ui-chart-controls/test/operators/renameOperator.test.ts
@@ -160,6 +160,44 @@ test('should add renameOperator if exists derived metrics', () => {
});
});
+test('should add renameOperator if isTimeComparisonValue without columns', () => {
+ [
+ ComparisonType.Difference,
+ ComparisonType.Ratio,
+ ComparisonType.Percentage,
+ ].forEach(type => {
+ expect(
+ renameOperator(
+ {
+ ...formData,
+ ...{
+ comparison_type: type,
+ time_compare: ['1 year ago'],
+ },
+ },
+ {
+ ...queryObject,
+ ...{
+ columns: [],
+ metrics: ['sum(val)', 'avg(val2)'],
+ },
+ },
+ ),
+ ).toEqual({
+ operation: 'rename',
+ options: {
+ columns: {
+ [`${type}__avg(val2)__avg(val2)__1 year ago`]:
+ 'avg(val2), 1 year ago',
+ [`${type}__sum(val)__sum(val)__1 year ago`]: 'sum(val), 1 year ago',
+ },
+ inplace: true,
+ level: 0,
+ },
+ });
+ });
+});
+
test('should add renameOperator if x_axis does not exist', () => {
expect(
renameOperator(