You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by gr...@apache.org on 2020/04/29 06:10:04 UTC
[incubator-superset] branch master updated: [explore view] add
partition as adhoc filter option (#9637)
This is an automated email from the ASF dual-hosted git repository.
graceguo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 735dcd2 [explore view] add partition as adhoc filter option (#9637)
735dcd2 is described below
commit 735dcd20022c90b74cdc58583505a44248c16e57
Author: Grace Guo <gr...@airbnb.com>
AuthorDate: Tue Apr 28 23:09:44 2020 -0700
[explore view] add partition as adhoc filter option (#9637)
* [explore view] add partition as adhoc option
* use adhocFilter Simple Tab
* simplify conditional check for custom adhoc filter operator
* add simple unit tests
---
...AdhocFilterEditPopoverSimpleTabContent_spec.jsx | 56 +++++++++++++++++++++
superset-frontend/src/explore/AdhocFilter.js | 24 +++++++--
.../explore/components/AdhocFilterEditPopover.jsx | 7 ++-
.../AdhocFilterEditPopoverSimpleTabContent.jsx | 58 +++++++++++++++-------
.../src/explore/components/AdhocFilterOption.jsx | 2 +
.../components/controls/AdhocFilterControl.jsx | 42 ++++++++++++++++
superset-frontend/src/explore/constants.js | 11 ++++
7 files changed, 176 insertions(+), 24 deletions(-)
diff --git a/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx b/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx
index d5079ce..1d1f376 100644
--- a/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx
+++ b/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx
@@ -52,6 +52,12 @@ const sumValueAdhocMetric = new AdhocMetric({
aggregate: AGGREGATES.SUM,
});
+const simpleCustomFilter = new AdhocFilter({
+ expressionType: EXPRESSION_TYPES.SIMPLE,
+ subject: 'ds',
+ operator: 'LATEST PARTITION',
+});
+
const options = [
{ type: 'VARCHAR(255)', column_name: 'source' },
{ type: 'VARCHAR(255)', column_name: 'target' },
@@ -155,6 +161,56 @@ describe('AdhocFilterEditPopoverSimpleTabContent', () => {
expect(wrapper.instance().isOperatorRelevant('LIKE')).toBe(false);
});
+ it('will show LATEST PARTITION operator', () => {
+ const { wrapper } = setup({
+ datasource: {
+ type: 'table',
+ datasource_name: 'table1',
+ schema: 'schema',
+ },
+ adhocFilter: simpleCustomFilter,
+ partitionColumn: 'ds',
+ });
+
+ expect(
+ wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'ds'),
+ ).toBe(true);
+ expect(
+ wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'value'),
+ ).toBe(false);
+ });
+
+ it('will generate custom sqlExpression for LATEST PARTITION operator', () => {
+ const testAdhocFilter = new AdhocFilter({
+ expressionType: EXPRESSION_TYPES.SIMPLE,
+ subject: 'ds',
+ });
+ const { wrapper, onChange } = setup({
+ datasource: {
+ type: 'table',
+ datasource_name: 'table1',
+ schema: 'schema',
+ },
+ adhocFilter: testAdhocFilter,
+ partitionColumn: 'ds',
+ });
+
+ wrapper.instance().onOperatorChange({ operator: 'LATEST PARTITION' });
+ expect(
+ onChange.lastCall.args[0].equals(
+ testAdhocFilter.duplicateWith({
+ subject: 'ds',
+ operator: 'LATEST PARTITION',
+ comparator: null,
+ clause: 'WHERE',
+ expressionType: 'SQL',
+ sqlExpression:
+ "ds = '{{ presto.latest_partition('schema.table1') }}' ",
+ }),
+ ),
+ ).toBe(true);
+ });
+
it('expands when its multi comparator input field expands', () => {
const { wrapper, onHeightChange } = setup();
diff --git a/superset-frontend/src/explore/AdhocFilter.js b/superset-frontend/src/explore/AdhocFilter.js
index 7520cec..0c84abc 100644
--- a/superset-frontend/src/explore/AdhocFilter.js
+++ b/superset-frontend/src/explore/AdhocFilter.js
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-import { MULTI_OPERATORS } from './constants';
+import { MULTI_OPERATORS, CUSTOM_OPERATORS } from './constants';
export const EXPRESSION_TYPES = {
SIMPLE: 'SIMPLE',
@@ -41,16 +41,22 @@ const OPERATORS_TO_SQL = {
regex: 'regex',
'IS NOT NULL': 'IS NOT NULL',
'IS NULL': 'IS NULL',
+ 'LATEST PARTITION': ({ datasource }) => {
+ return `= '{{ presto.latest_partition('${datasource.schema}.${datasource.datasource_name}') }}'`;
+ },
};
function translateToSql(adhocMetric, { useSimple } = {}) {
if (adhocMetric.expressionType === EXPRESSION_TYPES.SIMPLE || useSimple) {
const isMulti = MULTI_OPERATORS.indexOf(adhocMetric.operator) >= 0;
const subject = adhocMetric.subject;
- const operator = OPERATORS_TO_SQL[adhocMetric.operator];
+ const operator =
+ adhocMetric.operator && CUSTOM_OPERATORS.includes(adhocMetric.operator)
+ ? OPERATORS_TO_SQL[adhocMetric.operator](adhocMetric)
+ : OPERATORS_TO_SQL[adhocMetric.operator];
const comparator = Array.isArray(adhocMetric.comparator)
? adhocMetric.comparator.join("','")
- : adhocMetric.comparator;
+ : adhocMetric.comparator || '';
return `${subject} ${operator} ${isMulti ? "('" : ''}${comparator}${
isMulti ? "')" : ''
}`;
@@ -75,8 +81,16 @@ export default class AdhocFilter {
? adhocFilter.sqlExpression
: translateToSql(adhocFilter, { useSimple: true });
this.clause = adhocFilter.clause;
- this.subject = null;
- this.operator = null;
+ if (
+ adhocFilter.operator &&
+ CUSTOM_OPERATORS.includes(adhocFilter.operator)
+ ) {
+ this.subject = adhocFilter.subject;
+ this.operator = adhocFilter.operator;
+ } else {
+ this.subject = null;
+ this.operator = null;
+ }
this.comparator = null;
}
this.isExtra = !!adhocFilter.isExtra;
diff --git a/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx b/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx
index 7da098c..4aa19a2 100644
--- a/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx
@@ -39,6 +39,7 @@ const propTypes = {
]),
).isRequired,
datasource: PropTypes.object,
+ partitionColumn: PropTypes.string,
};
const startingWidth = 300;
@@ -117,6 +118,7 @@ export default class AdhocFilterEditPopover extends React.Component {
onClose,
onResize,
datasource,
+ partitionColumn,
...popoverProps
} = this.props;
@@ -141,9 +143,10 @@ export default class AdhocFilterEditPopover extends React.Component {
<AdhocFilterEditPopoverSimpleTabContent
adhocFilter={this.state.adhocFilter}
onChange={this.onAdhocFilterChange}
- options={this.props.options}
- datasource={this.props.datasource}
+ options={options}
+ datasource={datasource}
onHeightChange={this.adjustHeight}
+ partitionColumn={partitionColumn}
/>
</Tab>
<Tab
diff --git a/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx b/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx
index 3fb6be6..eadaa54 100644
--- a/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx
@@ -32,6 +32,8 @@ import {
DRUID_ONLY_OPERATORS,
HAVING_OPERATORS,
MULTI_OPERATORS,
+ CUSTOM_OPERATORS,
+ DISABLE_INPUT_OPERATORS,
} from '../constants';
import FilterDefinitionOption from './FilterDefinitionOption';
import OnPasteSelect from '../../components/OnPasteSelect';
@@ -50,6 +52,7 @@ const propTypes = {
).isRequired,
onHeightChange: PropTypes.func.isRequired,
datasource: PropTypes.object,
+ partitionColumn: PropTypes.string,
};
const defaultProps = {
@@ -63,6 +66,8 @@ function translateOperator(operator) {
return 'not equal to';
} else if (operator === OPERATORS.LIKE) {
return 'like';
+ } else if (operator === OPERATORS['LATEST PARTITION']) {
+ return 'use latest_partition template';
}
return operator;
}
@@ -124,10 +129,15 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
subject = option.saved_metric_name || option.label;
clause = CLAUSES.HAVING;
}
+ const { operator } = this.props.adhocFilter;
this.props.onChange(
this.props.adhocFilter.duplicateWith({
subject,
clause,
+ operator:
+ operator && this.isOperatorRelevant(operator, subject)
+ ? operator
+ : null,
expressionType: EXPRESSION_TYPES.SIMPLE,
}),
);
@@ -147,13 +157,26 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
? currentComparator[0]
: currentComparator;
}
- this.props.onChange(
- this.props.adhocFilter.duplicateWith({
- operator: operator && operator.operator,
- comparator: newComparator,
- expressionType: EXPRESSION_TYPES.SIMPLE,
- }),
- );
+
+ if (operator && CUSTOM_OPERATORS.includes(operator.operator)) {
+ this.props.onChange(
+ this.props.adhocFilter.duplicateWith({
+ subject: this.props.adhocFilter.subject,
+ clause: CLAUSES.WHERE,
+ operator: operator && operator.operator,
+ expressionType: EXPRESSION_TYPES.SQL,
+ datasource: this.props.datasource,
+ }),
+ );
+ } else {
+ this.props.onChange(
+ this.props.adhocFilter.duplicateWith({
+ operator: operator && operator.operator,
+ comparator: newComparator,
+ expressionType: EXPRESSION_TYPES.SIMPLE,
+ }),
+ );
+ }
}
onInputComparatorChange(event) {
@@ -220,7 +243,12 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
}
}
- isOperatorRelevant(operator) {
+ isOperatorRelevant(operator, subject) {
+ if (operator && CUSTOM_OPERATORS.includes(operator)) {
+ const { partitionColumn } = this.props;
+ return partitionColumn && subject && subject === partitionColumn;
+ }
+
return !(
(this.props.datasource.type === 'druid' &&
TABLE_ONLY_OPERATORS.indexOf(operator) >= 0) ||
@@ -282,7 +310,9 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
const operatorSelectProps = {
placeholder: t('%s operators(s)', Object.keys(OPERATORS).length),
options: Object.keys(OPERATORS)
- .filter(this.isOperatorRelevant)
+ .filter(operator =>
+ this.isOperatorRelevant(operator, adhocFilter.subject),
+ )
.map(operator => ({ operator })),
value: adhocFilter.operator,
onChange: this.onOperatorChange,
@@ -317,10 +347,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
showHeader={false}
noResultsText={t('type a value here')}
refFunc={this.multiComparatorRef}
- disabled={
- adhocFilter.operator === 'IS NOT NULL' ||
- adhocFilter.operator === 'IS NULL'
- }
+ disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)}
/>
) : (
<input
@@ -330,10 +357,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
value={adhocFilter.comparator || ''}
className="form-control input-sm"
placeholder={t('Filter value')}
- disabled={
- adhocFilter.operator === 'IS NOT NULL' ||
- adhocFilter.operator === 'IS NULL'
- }
+ disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)}
/>
)}
</FormGroup>
diff --git a/superset-frontend/src/explore/components/AdhocFilterOption.jsx b/superset-frontend/src/explore/components/AdhocFilterOption.jsx
index 10979c2..ea17315 100644
--- a/superset-frontend/src/explore/components/AdhocFilterOption.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterOption.jsx
@@ -38,6 +38,7 @@ const propTypes = {
]),
).isRequired,
datasource: PropTypes.object,
+ partitionColumn: PropTypes.string,
};
export default class AdhocFilterOption extends React.PureComponent {
@@ -80,6 +81,7 @@ export default class AdhocFilterOption extends React.PureComponent {
onClose={this.closeFilterEditOverlay}
options={this.props.options}
datasource={this.props.datasource}
+ partitionColumn={this.props.partitionColumn}
/>
);
return (
diff --git a/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx b/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx
index 64035b6..79b32ac 100644
--- a/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx
+++ b/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx
@@ -21,6 +21,8 @@ import PropTypes from 'prop-types';
import VirtualizedSelect from 'react-virtualized-select';
import { t } from '@superset-ui/translation';
+import { SupersetClient } from '@superset-ui/connection';
+
import ControlHeader from '../ControlHeader';
import adhocFilterType from '../../propTypes/adhocFilterType';
import adhocMetricType from '../../propTypes/adhocMetricType';
@@ -90,6 +92,46 @@ export default class AdhocFilterControl extends React.Component {
};
}
+ componentDidMount() {
+ const { datasource } = this.props;
+ if (datasource && datasource.type === 'table') {
+ const dbId = datasource.database ? datasource.database.id : null;
+ const datasourceName = datasource.datasource_name;
+ const datasourceSchema = datasource.schema;
+
+ if (dbId && datasourceName && datasourceSchema) {
+ SupersetClient.get({
+ endpoint: `/superset/extra_table_metadata/${dbId}/${datasourceName}/${datasourceSchema}/`,
+ }).then(
+ ({ json }) => {
+ if (json && json.partitions) {
+ const partitions = json.partitions;
+ // for now only show latest_partition option
+ // when table datasource has only 1 partition key.
+ if (
+ partitions &&
+ partitions.cols &&
+ Object.keys(partitions.cols).length === 1
+ ) {
+ const partitionColumn = partitions.cols[0];
+ this.valueRenderer = adhocFilter => (
+ <AdhocFilterOption
+ adhocFilter={adhocFilter}
+ onFilterEdit={this.onFilterEdit}
+ options={this.state.options}
+ datasource={this.props.datasource}
+ partitionColumn={partitionColumn}
+ />
+ );
+ }
+ }
+ },
+ // no error handler, in case of error do not show partition option
+ );
+ }
+ }
+ }
+
UNSAFE_componentWillReceiveProps(nextProps) {
if (
this.props.columns !== nextProps.columns ||
diff --git a/superset-frontend/src/explore/constants.js b/superset-frontend/src/explore/constants.js
index 8f1b395..de99199 100644
--- a/superset-frontend/src/explore/constants.js
+++ b/superset-frontend/src/explore/constants.js
@@ -40,6 +40,7 @@ export const OPERATORS = {
regex: 'regex',
'IS NOT NULL': 'IS NOT NULL',
'IS NULL': 'IS NULL',
+ 'LATEST PARTITION': 'LATEST PARTITION',
};
export const TABLE_ONLY_OPERATORS = [OPERATORS.LIKE];
@@ -53,6 +54,16 @@ export const HAVING_OPERATORS = [
OPERATORS['<='],
];
export const MULTI_OPERATORS = [OPERATORS.in, OPERATORS['not in']];
+// CUSTOM_OPERATORS will show operator in simple mode,
+// but will generate customized sqlExpression
+export const CUSTOM_OPERATORS = [OPERATORS['LATEST PARTITION']];
+// DISABLE_INPUT_OPERATORS will disable filter value input
+// in adhocFilter control
+export const DISABLE_INPUT_OPERATORS = [
+ OPERATORS['IS NOT NULL'],
+ OPERATORS['IS NULL'],
+ OPERATORS['LATEST PARTITION'],
+];
export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_distinct)__.*$/i;
export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;