You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@iotdb.apache.org by GitBox <gi...@apache.org> on 2022/09/27 14:35:56 UTC
[GitHub] [iotdb] liuminghui233 commented on a diff in pull request #7442: [IOTDB-4524] Refactor analyzer for query statement
liuminghui233 commented on code in PR #7442:
URL: https://github.com/apache/iotdb/pull/7442#discussion_r981331497
##########
server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java:
##########
@@ -204,317 +203,151 @@ public Analysis visitQuery(QueryStatement queryStatement, MPPQueryContext contex
}
// extract global time filter from query filter and determine if there is a value filter
- Pair<Filter, Boolean> resultPair = analyzeGlobalTimeFilter(queryStatement);
- Filter globalTimeFilter = resultPair.left;
- boolean hasValueFilter = resultPair.right;
- analysis.setGlobalTimeFilter(globalTimeFilter);
- analysis.setHasValueFilter(hasValueFilter);
+ analyzeGlobalTimeFilter(analysis, queryStatement);
if (queryStatement.isLastQuery()) {
- if (hasValueFilter) {
+ if (analysis.hasValueFilter()) {
throw new SemanticException("Only time filters are supported in LAST query");
}
- analysis.setMergeOrderParameter(analyzeOrderBy(queryStatement));
+ analyzeOrderBy(analysis, queryStatement);
return analyzeLast(analysis, schemaTree.getAllMeasurement(), schemaTree);
}
- // Example 1: select s1, s1 + s2 as t, udf(udf(s1)) from root.sg.d1
- // outputExpressions: [<root.sg.d1.s1,null>, <root.sg.d1.s1 + root.sg.d1.s2,t>,
- // <udf(udf(root.sg.d1.s1)),null>]
- // transformExpressions: [root.sg.d1.s1, root.sg.d1.s1 + root.sg.d1.s2,
- // udf(udf(root.sg.d1.s1))]
- // sourceExpressions: {root.sg.d1 -> [root.sg.d1.s1, root.sg.d1.s2]}
- //
- // Example 2: select s1, s2, s3 as t from root.sg.* align by device
- // outputExpressions: [<s1,null>, <s2,null>, <s1,t>]
- // transformExpressions: [root.sg.d1.s1, root.sg.d1.s2, root.sg.d1.s3,
- // root.sg.d2.s1, root.sg.d2.s2]
- // sourceExpressions: {root.sg.d1 -> [root.sg.d1.s1, root.sg.d1.s2, root.sg.d1.s2],
- // root.sg.d2 -> [root.sg.d2.s1, root.sg.d2.s2]}
- //
- // Example 3: select sum(s1) + 1 as t, count(s2) from root.sg.d1
- // outputExpressions: [<sum(root.sg.d1.s1) + 1,t>, <count(root.sg.d1.s2),t>]
- // transformExpressions: [sum(root.sg.d1.s1) + 1, count(root.sg.d1.s2)]
- // aggregationExpressions: {root.sg.d1 -> [sum(root.sg.d1.s1), count(root.sg.d1.s2)]}
- // sourceExpressions: {root.sg.d1 -> [sum(root.sg.d1.s1), count(root.sg.d1.s2)]}
- //
- // Example 4: select sum(s1) + 1 as t, count(s2) from root.sg.d1 where s1 > 1
- // outputExpressions: [<sum(root.sg.d1.s1) + 1,t>, <count(root.sg.d1.s2),t>]
- // transformExpressions: [sum(root.sg.d1.s1) + 1, count(root.sg.d1.s2)]
- // aggregationExpressions: {root.sg.d1 -> [sum(root.sg.d1.s1), count(root.sg.d1.s2)]}
- // sourceExpressions: {root.sg.d1 -> [root.sg.d1.s1, root.sg.d1.s2]}
List<Pair<Expression, String>> outputExpressions;
if (queryStatement.isAlignByDevice()) {
- Map<String, Set<Expression>> deviceToTransformExpressions = new HashMap<>();
- Expression deviceExpression =
- new TimeSeriesOperand(new MeasurementPath(COLUMN_DEVICE, TSDataType.TEXT));
- Set<Expression> transformInput = new LinkedHashSet<>();
- transformInput.add(deviceExpression);
- Set<Expression> transformOutput = new LinkedHashSet<>();
- transformOutput.add(deviceExpression);
-
- // all selected device
- Set<PartialPath> deviceList = analyzeFrom(queryStatement, schemaTree);
-
- Map<String, Set<String>> deviceToMeasurementsMap = new HashMap<>();
- outputExpressions =
- analyzeSelect(
- analysis,
- schemaTree,
- deviceList,
- deviceToTransformExpressions,
- deviceToMeasurementsMap,
- transformInput);
-
- transformOutput.addAll(
- outputExpressions.stream().map(Pair::getLeft).collect(Collectors.toList()));
-
- if (queryStatement.hasHaving()) {
- Expression havingExpression =
- analyzeHaving(
- analysis,
- schemaTree,
- deviceList,
- deviceToTransformExpressions,
- deviceToMeasurementsMap,
- transformInput);
- analyzeExpression(analysis, havingExpression);
-
- // used for planFilter after planDeviceView
- analysis.setHavingExpression(havingExpression);
- }
-
- List<String> allMeasurements =
- transformInput.stream()
- .map(Expression::getExpressionString)
- .collect(Collectors.toList());
-
- Map<String, List<Integer>> deviceToMeasurementIndexesMap = new HashMap<>();
- for (String deviceName : deviceToMeasurementsMap.keySet()) {
- List<String> measurementsUnderDevice =
- new ArrayList<>(deviceToMeasurementsMap.get(deviceName));
- List<Integer> indexes = new ArrayList<>();
- for (String measurement : measurementsUnderDevice) {
- indexes.add(allMeasurements.indexOf(measurement));
- }
- deviceToMeasurementIndexesMap.put(deviceName, indexes);
- }
- analysis.setDeviceToMeasurementIndexesMap(deviceToMeasurementIndexesMap);
-
- Map<String, Set<Expression>> deviceToSourceExpressions = new HashMap<>();
- boolean isValueFilterAggregation = queryStatement.isAggregationQuery() && hasValueFilter;
-
- Map<String, Boolean> deviceToIsRawDataSource = new HashMap<>();
+ Set<PartialPath> deviceSet = analyzeFrom(queryStatement, schemaTree);
+ outputExpressions = analyzeSelect(analysis, queryStatement, schemaTree, deviceSet);
Map<String, Set<Expression>> deviceToAggregationExpressions = new HashMap<>();
- Map<String, Set<Expression>> deviceToAggregationTransformExpressions = new HashMap<>();
- for (String deviceName : deviceToTransformExpressions.keySet()) {
- Set<Expression> transformExpressions = deviceToTransformExpressions.get(deviceName);
- Set<Expression> aggregationExpressions = new LinkedHashSet<>();
- Set<Expression> aggregationTransformExpressions = new LinkedHashSet<>();
-
- boolean isHasRawDataInputAggregation = false;
- if (queryStatement.isAggregationQuery()) {
- // true if nested expressions and UDFs exist in aggregation function
- // i.e. select sum(s1 + 1) from root.sg.d1 align by device
- isHasRawDataInputAggregation =
- analyzeAggregation(
- transformExpressions, aggregationExpressions, aggregationTransformExpressions);
- deviceToAggregationExpressions.put(deviceName, aggregationExpressions);
- deviceToAggregationTransformExpressions.put(
- deviceName, aggregationTransformExpressions);
- }
+ analyzeHaving(
+ analysis, queryStatement, schemaTree, deviceSet, deviceToAggregationExpressions);
+ analyzeDeviceToAggregation(analysis, queryStatement, deviceToAggregationExpressions);
+ analysis.setDeviceToAggregationExpressions(deviceToAggregationExpressions);
- boolean isRawDataSource =
- !queryStatement.isAggregationQuery()
- || isValueFilterAggregation
- || isHasRawDataInputAggregation;
+ analyzeDeviceToWhere(analysis, queryStatement, schemaTree, deviceSet);
+ analyzeDeviceToSourceTransform(analysis, queryStatement);
- for (Expression expression : transformExpressions) {
- updateSource(
- expression,
- deviceToSourceExpressions.computeIfAbsent(deviceName, key -> new LinkedHashSet<>()),
- isRawDataSource);
- }
- deviceToIsRawDataSource.put(deviceName, isRawDataSource);
- }
- analysis.setDeviceToAggregationExpressions(deviceToAggregationExpressions);
- analysis.setDeviceToAggregationTransformExpressions(
- deviceToAggregationTransformExpressions);
- analysis.setDeviceToIsRawDataSource(deviceToIsRawDataSource);
-
- if (queryStatement.getWhereCondition() != null) {
- Map<String, Expression> deviceToQueryFilter = new HashMap<>();
- Iterator<PartialPath> deviceIterator = deviceList.iterator();
- while (deviceIterator.hasNext()) {
- PartialPath devicePath = deviceIterator.next();
- Expression queryFilter = null;
- try {
- queryFilter = analyzeWhereSplitByDevice(queryStatement, devicePath, schemaTree);
- } catch (SemanticException e) {
- if (e instanceof MeasurementNotExistException) {
- logger.warn(e.getMessage());
- deviceIterator.remove();
- deviceToSourceExpressions.remove(devicePath.getFullPath());
- continue;
- }
- throw e;
- }
- deviceToQueryFilter.put(devicePath.getFullPath(), queryFilter);
- analyzeExpression(analysis, queryFilter);
- updateSource(
- queryFilter,
- deviceToSourceExpressions.computeIfAbsent(
- devicePath.getFullPath(), key -> new LinkedHashSet<>()),
- true);
- }
- analysis.setDeviceToQueryFilter(deviceToQueryFilter);
- }
- analysis.setTransformInput(transformInput);
- analysis.setTransformOutput(transformOutput);
- analysis.setDeviceToSourceExpressions(deviceToSourceExpressions);
- analysis.setDeviceToTransformExpressions(deviceToTransformExpressions);
+ analyzeDeviceToSource(analysis, queryStatement);
+ analyzeDeviceView(analysis, queryStatement, outputExpressions);
} else {
- outputExpressions = analyzeSelect(analysis, schemaTree);
- Set<Expression> transformExpressions =
+ outputExpressions = analyzeSelect(analysis, queryStatement, schemaTree);
+
+ analyzeHaving(analysis, queryStatement, schemaTree);
+ analyzeGroupByLevel(analysis, queryStatement, outputExpressions);
+
+ Set<Expression> selectExpressions =
outputExpressions.stream()
.map(Pair::getLeft)
.collect(Collectors.toCollection(LinkedHashSet::new));
+ analysis.setSelectExpressions(selectExpressions);
- // cast functionName to lowercase in havingExpression
- Expression havingExpression =
- queryStatement.hasHaving()
- ? ExpressionAnalyzer.removeAliasFromExpression(
- queryStatement.getHavingCondition().getPredicate())
- : null;
-
- // get removeWildcard Expressions in Having
- // used to analyzeAggregation in Having expression and updateSource
- Set<Expression> transformExpressionsInHaving =
- queryStatement.hasHaving()
- ? new HashSet<>(
- ExpressionAnalyzer.removeWildcardInFilter(
- havingExpression,
- queryStatement.getFromComponent().getPrefixPaths(),
- schemaTree,
- false))
- : null;
-
- if (queryStatement.isGroupByLevel()) {
- // map from grouped expression to set of input expressions
- Map<Expression, Expression> rawPathToGroupedPathMap = new HashMap<>();
- Map<Expression, Set<Expression>> groupByLevelExpressions =
- analyzeGroupByLevel(
- analysis, outputExpressions, transformExpressions, rawPathToGroupedPathMap);
- analysis.setGroupByLevelExpressions(groupByLevelExpressions);
- analysis.setRawPathToGroupedPathMap(rawPathToGroupedPathMap);
- }
-
- // true if nested expressions and UDFs exist in aggregation function
- // i.e. select sum(s1 + 1) from root.sg.d1
- boolean isHasRawDataInputAggregation = false;
- if (queryStatement.isAggregationQuery()) {
- Set<Expression> aggregationExpressions = new HashSet<>();
- Set<Expression> aggregationTransformExpressions = new HashSet<>();
- List<Expression> aggregationExpressionsInHaving =
- new ArrayList<>(); // as input of GroupByLevelController
- isHasRawDataInputAggregation =
- analyzeAggregation(
- transformExpressions, aggregationExpressions, aggregationTransformExpressions);
- if (queryStatement.hasHaving()) {
- isHasRawDataInputAggregation |=
- analyzeAggregationInHaving(
- transformExpressionsInHaving,
- aggregationExpressionsInHaving,
- aggregationExpressions,
- aggregationTransformExpressions);
-
- havingExpression = // construct Filter from Having
- analyzeHaving(
- analysis,
- analysis.getGroupByLevelExpressions(),
- transformExpressionsInHaving,
- aggregationExpressionsInHaving);
- analyzeExpression(analysis, havingExpression);
- analysis.setHavingExpression(havingExpression);
- }
- analysis.setAggregationExpressions(aggregationExpressions);
- analysis.setAggregationTransformExpressions(aggregationTransformExpressions);
- }
-
- // generate sourceExpression according to transformExpressions
- Set<Expression> sourceExpressions = new HashSet<>();
- boolean isValueFilterAggregation = queryStatement.isAggregationQuery() && hasValueFilter;
- boolean isRawDataSource =
- !queryStatement.isAggregationQuery()
- || isValueFilterAggregation
- || isHasRawDataInputAggregation;
- for (Expression expression : transformExpressions) {
- updateSource(expression, sourceExpressions, isRawDataSource);
- }
-
- if (queryStatement.hasHaving()) {
- for (Expression expression : transformExpressionsInHaving) {
- analyzeExpression(analysis, expression);
- updateSource(expression, sourceExpressions, isRawDataSource);
- }
- }
+ analyzeAggregation(analysis, queryStatement);
- if (queryStatement.getWhereCondition() != null) {
- Expression queryFilter = analyzeWhere(queryStatement, schemaTree);
+ analyzeWhere(analysis, queryStatement, schemaTree);
+ analyzeSourceTransform(analysis, queryStatement);
- // update sourceExpression according to queryFilter
- analyzeExpression(analysis, queryFilter);
- updateSource(queryFilter, sourceExpressions, isRawDataSource);
- analysis.setQueryFilter(queryFilter);
- }
- analysis.setRawDataSource(isRawDataSource);
- analysis.setSourceExpressions(sourceExpressions);
- analysis.setTransformExpressions(transformExpressions);
+ analyzeSource(analysis, queryStatement);
}
- if (queryStatement.isGroupByTime()) {
- GroupByTimeComponent groupByTimeComponent = queryStatement.getGroupByTimeComponent();
- if ((groupByTimeComponent.isIntervalByMonth()
- || groupByTimeComponent.isSlidingStepByMonth())
- && queryStatement.getResultTimeOrder() == Ordering.DESC) {
- throw new SemanticException("Group by month doesn't support order by time desc now.");
- }
- analysis.setGroupByTimeParameter(new GroupByTimeParameter(groupByTimeComponent));
- }
+ analyzeWindowFunction(analysis, queryStatement);
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org