You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@superset.apache.org by GitBox <gi...@apache.org> on 2021/02/03 23:08:34 UTC
[GitHub] [superset] villebro commented on a change in pull request #12933: fix(viz): improve dtype inference logic
villebro commented on a change in pull request #12933:
URL: https://github.com/apache/superset/pull/12933#discussion_r569815847
##########
File path: superset/utils/core.py
##########
@@ -1401,19 +1407,48 @@ def get_column_names_from_metrics(metrics: List[Metric]) -> List[str]:
return columns
-def serialize_pandas_dtypes(dtypes: List[np.dtype]) -> List[GenericDataType]:
- """Serialize pandas/numpy dtypes to JavaScript types"""
- mapping = {
- "object": GenericDataType.STRING,
- "category": GenericDataType.STRING,
- "datetime64[ns]": GenericDataType.TEMPORAL,
- "int64": GenericDataType.NUMERIC,
- "in32": GenericDataType.NUMERIC,
- "float64": GenericDataType.NUMERIC,
- "float32": GenericDataType.NUMERIC,
- "bool": GenericDataType.BOOLEAN,
+def extract_dataframe_dtypes(df: pd.DataFrame) -> List[GenericDataType]:
+ """Serialize pandas/numpy dtypes to generic types"""
+
+ def _check_dtype(series: pd.Series) -> Optional[GenericDataType]:
+ if is_numeric_dtype(series):
+ return GenericDataType.NUMERIC
+ elif is_bool_dtype(series):
+ return GenericDataType.BOOLEAN
+ elif is_datetime64_any_dtype(series):
+ return GenericDataType.TEMPORAL
+ return None
+
+ type_map: Dict[Type[Any], GenericDataType] = {
+ date: GenericDataType.TEMPORAL,
+ datetime: GenericDataType.TEMPORAL,
+ pd.Timestamp: GenericDataType.TEMPORAL,
+ pd.DatetimeTZDtype: GenericDataType.TEMPORAL,
+ str: GenericDataType.STRING,
+ int: GenericDataType.NUMERIC,
+ float: GenericDataType.NUMERIC,
+ bool: GenericDataType.BOOLEAN,
}
- return [mapping.get(str(x), GenericDataType.STRING) for x in dtypes]
+
+ generic_dtypes: List[GenericDataType] = []
+ for idx, column in enumerate(df.columns):
+ generic_dtype = GenericDataType.STRING
+ series = df[column]
+ if is_object_dtype(df[column]):
+ for row_idx, cell in enumerate(series):
+ inferred_generic_type = type_map.get(type(cell))
+ if inferred_generic_type is not None:
+ generic_dtype = inferred_generic_type
+ break
+ if row_idx > 1000:
+ break
Review comment:
I'm not a big fan of magic numbers, but this code is not expected to be long-lived.
##########
File path: tests/utils_tests.py
##########
@@ -1066,3 +1069,35 @@ def test_get_form_data_token(self):
assert get_form_data_token({"token": "token_abcdefg1"}) == "token_abcdefg1"
generated_token = get_form_data_token({})
assert re.match(r"^token_[a-z0-9]{8}$", generated_token) is not None
+
+ def test_extract_dataframe_dtypes(self):
+ df = pd.DataFrame(
+ data={
+ "dt": [date(2021, 2, 4), date(2021, 2, 4)],
+ "dttm": [datetime(2021, 2, 4, 1, 1, 1), datetime(2021, 2, 4, 1, 1, 1)],
+ "str": ["foo", "foo"],
+ "int": [1, 1],
+ "float": [0.5, 0.5],
+ "obj": [{"a": 1}, {"a": 1}],
+ "dt_null": [None, date(2021, 2, 4)],
+ "dttm_null": [None, datetime(2021, 2, 4, 1, 1, 1)],
+ "str_null": [None, "foo"],
+ "int_null": [None, 1],
+ "float_null": [None, 0.5],
+ "obj_null": [None, {"a": 1}],
+ }
+ )
+ assert extract_dataframe_dtypes(df) == [
+ GenericDataType.TEMPORAL,
+ GenericDataType.TEMPORAL,
+ GenericDataType.STRING,
+ GenericDataType.NUMERIC,
+ GenericDataType.NUMERIC,
+ GenericDataType.STRING,
+ GenericDataType.TEMPORAL,
+ GenericDataType.TEMPORAL,
+ GenericDataType.STRING,
+ GenericDataType.NUMERIC,
+ GenericDataType.NUMERIC,
+ GenericDataType.STRING,
+ ]
Review comment:
Here we're making sure that all main datatypes are correctly identified both with and without null values.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@superset.apache.org
For additional commands, e-mail: notifications-help@superset.apache.org