You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@superset.apache.org by GitBox <gi...@apache.org> on 2019/04/30 17:08:25 UTC
[GitHub] [incubator-superset] DiggidyDave commented on a change in pull
request #7413: feature: see Presto row and array data types
DiggidyDave commented on a change in pull request #7413: feature: see Presto row and array data types
URL: https://github.com/apache/incubator-superset/pull/7413#discussion_r279849082
##########
File path: superset/db_engine_specs.py
##########
@@ -814,6 +822,178 @@ def get_view_names(cls, inspector, schema):
"""
return []
+ @classmethod
+ def _create_column_info(cls, column: RowProxy, name: str, data_type: str) -> dict:
+ """
+ Create column info object
+ :param column: column object
+ :param name: column name
+ :param data_type: column data type
+ :return: column info object
+ """
+ return {
+ 'name': name,
+ 'type': data_type,
+ # newer Presto no longer includes this column
+ 'nullable': getattr(column, 'Null', True),
+ 'default': None,
+ }
+
+ @classmethod
+ def _get_full_name(cls, names: list) -> str:
+ """
+ Get the full column name
+ :param names: list of all individual column names
+ :return: full column name
+ """
+ return '.'.join(row_type[0] for row_type in names if row_type[0] is not None)
+
+ @classmethod
+ def _has_nested_data_types(cls, component_type: str) -> bool:
+ """
+ Check if string contains a data type. We determine if there is a data type by
+ whitespace or multiple data types by commas
+ :param component_type: data type
+ :return: boolean
+ """
+ comma_regex = r',(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'
+ white_space_regex = r'\s(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'
+ return re.search(comma_regex, component_type) is not None \
+ or re.search(white_space_regex, component_type) is not None
+
+ @classmethod
+ def _split_data_type(cls, data_type: str, delimiter: str) -> list:
+ """
+ Split data type based on given delimiter. Do not split the string if the
+ delimiter is enclosed in quotes
+ :param data_type: data type
+ :param delimiter: string separator (i.e. open parenthesis, closed parenthesis,
+ comma, whitespace)
+ :return:list of strings after breaking it by the delimiter
+ """
+ return re.split(
+ r'{}(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'.format(delimiter), data_type)
+
+ @classmethod
+ def _parse_structural_column(cls, column: RowProxy, result: list) -> None:
+ """
+ Parse a row or array column
+ :param column: column
+ :param result: list tracking the results
+ """
+ full_data_type = '{} {}'.format(column.Column, column.Type)
+ # split on open parenthesis ( to get the structural
+ # data type and its component types
+ data_types = cls._split_data_type(full_data_type, r'\(')
+ stack: list = []
+ for data_type in data_types:
+ # split on closed parenthesis ) to track which component
+ # types belong to what structural data type
+ inner_types = cls._split_data_type(data_type, r'\)')
+ for inner_type in inner_types:
+ # We have finished parsing multiple structural data types
+ if not inner_type and len(stack) > 0:
+ stack.pop()
+ elif cls._has_nested_data_types(inner_type):
+ # split on comma , to get individual data types
+ single_fields = cls._split_data_type(inner_type, ', ')
+ for single_field in single_fields:
+ # If component type starts with a comma, the first single field
+ # will be an empty string. Disregard this empty string.
+ if not single_field:
+ continue
+ # split on whitespace to get field name and data type
+ field_info = cls._split_data_type(single_field, r'\s')
+ # check if there is a structural data type within
+ # overall structural data type
+ if field_info[1] == 'array' or field_info[1] == 'row':
+ stack.append((field_info[0], field_info[1]))
+ full_parent_path = cls._get_full_name(stack)
+ result.append(cls._create_column_info(
+ column, full_parent_path,
+ presto_type_map[field_info[1]]()))
+ else: # otherwise this field is a basic data type
+ full_parent_path = cls._get_full_name(stack)
+ column_name = '{}.{}'.format(full_parent_path, field_info[0])
+ result.append(cls._create_column_info(
+ column, column_name, presto_type_map[field_info[1]]()))
+ # If the component type ends with a structural data type, do not pop
+ # the stack. We have run across a structural data type within the
+ # overall structural data type. Otherwise, we have completely parsed
+ # through the entire structural data type and can move on.
+ if not (inner_type.endswith('array') or inner_type.endswith('row')):
+ stack.pop()
+ # We have an array of row objects (i.e. array(row(...)))
+ elif 'array' == inner_type or 'row' == inner_type:
+ # Push a dummy object to represent the structural data type
+ stack.append((None, inner_type))
+ # We have an array of a basic data types(i.e. array(varchar)).
+ elif len(stack) > 0:
+ # Because it is an array of a basic data type. We have finished
+ # parsing the structural data type and can move on.
+ stack.pop()
+
+ @classmethod
+ def _show_columns(cls, inspector: Inspector, table_name: str, schema: str) -> list:
+ """
+ Show presto column names
+ :param inspector: object that performs database schema inspection
+ :param table_name: table name
+ :param schema: schema name
+ :return: list of column objects
+ """
+ quote = inspector.engine.dialect.identifier_preparer.quote_identifier
+ full_table = quote(table_name)
+ if schema:
+ full_table = '{}.{}'.format(quote(schema), full_table)
+ columns = inspector.bind.execute('SHOW COLUMNS FROM {}'.format(full_table))
+ return columns
+
+ @classmethod
+ def get_columns(cls, inspector: Inspector, table_name: str, schema: str) -> list:
+ """
+ Get columns from a Presto data source. This includes handling row and
+ array data types
+ :param inspector: object that performs database schema inspection
+ :param table_name: table name
+ :param schema: schema name
+ :return: a list of results that contain column info
+ (i.e. column name and data type)
+ """
+ columns = cls._show_columns(inspector, table_name, schema)
+ result: list = []
+ for column in columns:
+ try:
+ # parse column if it is a row or array
+ if 'array' in column.Type or 'row' in column.Type:
+ cls._parse_structural_column(column, result)
+ continue
+ else: # otherwise column is a basic data type
+ column_type = presto_type_map[column.Type]()
+ except KeyError:
+ print('Did not recognize type {} of column {}'.format(
Review comment:
@khtruong fix this in a new PR against `lyft-release-sp8`... we can bring both commits over in one merge.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@superset.apache.org
For additional commands, e-mail: notifications-help@superset.apache.org