You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/09/18 13:01:43 UTC
[iotdb] branch master updated: [IOTDB-4319] move transform above the DevieView and fix the case sensitive issue (#7308)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f140313981 [IOTDB-4319] move transform above the DevieView and fix the case sensitive issue (#7308)
f140313981 is described below
commit f140313981fce92002cd1532fbebc27c0fb40a7a
Author: Weihao Li <60...@users.noreply.github.com>
AuthorDate: Sun Sep 18 21:01:36 2022 +0800
[IOTDB-4319] move transform above the DevieView and fix the case sensitive issue (#7308)
---
.../db/it/alignbydevice/IoTDBAlignByDeviceIT.java | 2 +-
.../apache/iotdb/db/it/groupby/IoTDBHavingIT.java | 51 ++++
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 35 ++-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 268 ++++++++++++++-------
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 4 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 23 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 42 ++--
7 files changed, 286 insertions(+), 139 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
index 7ba15e7c13..6e9abfa9fc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
@@ -475,7 +475,7 @@ public class IoTDBAlignByDeviceIT {
"103,root.vehicle.d0,99,",
"104,root.vehicle.d0,90,",
"105,root.vehicle.d0,99,",
- "946684800000,root.vehicle.d0,null",
+ "946684800000,root.vehicle.d0,null"
};
try (Connection connection = EnvFactory.getEnv().getConnection();
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBHavingIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBHavingIT.java
index 79478a981b..171cbba648 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBHavingIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBHavingIT.java
@@ -37,6 +37,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualWithDescOrder
import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
import static org.apache.iotdb.itbase.constant.TestConstant.avg;
import static org.apache.iotdb.itbase.constant.TestConstant.count;
+import static org.apache.iotdb.itbase.constant.TestConstant.lastValue;
import static org.apache.iotdb.itbase.constant.TestConstant.sum;
@RunWith(IoTDBTestRunner.class)
@@ -260,4 +261,54 @@ public class IoTDBHavingIT {
expectedHeader,
retArray);
}
+
+ @Test
+ public void caseSensitiveHavingTest() {
+ String[] expectedHeader =
+ new String[] {
+ TIMESTAMP_STR,
+ lastValue("root.test.sg5.s1"),
+ lastValue("root.test.sg1.s1"),
+ lastValue("root.test.sg2.s1"),
+ };
+ String[] retArray =
+ new String[] {
+ "1,true,true,true,", "5,null,true,true,", "7,null,true,true,", "9,null,true,true,"
+ };
+ resultSetEqualTest(
+ "select last_value(s1) from root.** "
+ + "GROUP BY ([1,11),2ms) "
+ + "Having LAST_VALUE(s2) > 0 ",
+ expectedHeader,
+ retArray);
+
+ expectedHeader = new String[] {TIMESTAMP_STR, lastValue("root.test.*.s1")};
+ retArray = new String[] {"1,true,", "5,true,", "7,true,", "9,true,"};
+ resultSetEqualTest(
+ "select last_value(s1) from root.** "
+ + "GROUP BY ([1,11),2ms), level=1 "
+ + "Having LAST_VALUE(s2) > 0 ",
+ expectedHeader,
+ retArray);
+
+ expectedHeader = new String[] {TIMESTAMP_STR, "Device", lastValue("s1")};
+ retArray =
+ new String[] {
+ "1,root.test.sg1,true,",
+ "5,root.test.sg1,true,",
+ "7,root.test.sg1,true,",
+ "9,root.test.sg1,true,",
+ "1,root.test.sg2,true,",
+ "5,root.test.sg2,true,",
+ "7,root.test.sg2,true,",
+ "9,root.test.sg2,true,"
+ };
+ resultSetEqualTest(
+ "select last_value(s1) from root.** "
+ + "GROUP BY ([1,11),2ms) "
+ + "Having LAST_VALUE(s2) > 0 "
+ + "align by device",
+ expectedHeader,
+ retArray);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
index d45d98e22c..1c1b185224 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
@@ -86,8 +86,6 @@ public class Analysis {
private Expression queryFilter;
- private Expression havingExpression;
-
// map from grouped path name to list of input aggregation in `GROUP BY LEVEL` clause
private Map<Expression, Set<Expression>> groupByLevelExpressions;
@@ -100,6 +98,10 @@ public class Analysis {
// Query Analysis (used in ALIGN BY DEVICE)
/////////////////////////////////////////////////////////////////////////////////////////////////
+ // used to planTransform after planDeviceView
+ Set<Expression> transformInput;
+ Set<Expression> transformOutput;
+
// map from device name to series/aggregation under this device
private Map<String, Set<Expression>> deviceToSourceExpressions;
@@ -115,9 +117,6 @@ public class Analysis {
// map from device name to query filter under this device
private Map<String, Expression> deviceToQueryFilter;
- // map from device name to havingExpression under this device
- private Map<String, Expression> deviceToHavingExpression;
-
// e.g. [s1,s2,s3] is query, but [s1, s3] exists in device1, then device1 -> [1, 3], s1 is 1 but
// not 0 because device is the first column
private Map<String, List<Integer>> deviceToMeasurementIndexesMap;
@@ -133,6 +132,8 @@ public class Analysis {
// indicate is there a value filter
private boolean hasValueFilter = false;
+ private Expression havingExpression;
+
// true if nested expressions and UDFs exist in aggregation function
private boolean isHasRawDataInputAggregation;
@@ -310,14 +311,6 @@ public class Analysis {
this.havingExpression = havingExpression;
}
- public Map<String, Expression> getDeviceToHavingExpression() {
- return deviceToHavingExpression;
- }
-
- public void setDeviceToHavingExpression(Map<String, Expression> deviceTohavingExpression) {
- this.deviceToHavingExpression = deviceTohavingExpression;
- }
-
public void setGroupByTimeParameter(GroupByTimeParameter groupByTimeParameter) {
this.groupByTimeParameter = groupByTimeParameter;
}
@@ -371,6 +364,22 @@ public class Analysis {
this.transformExpressions = transformExpressions;
}
+ public Set<Expression> getTransformInput() {
+ return transformInput;
+ }
+
+ public void setTransformInput(Set<Expression> transformInput) {
+ this.transformInput = transformInput;
+ }
+
+ public Set<Expression> getTransformOutput() {
+ return transformOutput;
+ }
+
+ public void setTransformOutput(Set<Expression> transformOutput) {
+ this.transformOutput = transformOutput;
+ }
+
public Map<String, Set<Expression>> getDeviceToSourceExpressions() {
return deviceToSourceExpressions;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index 391e44d48a..980ccbeac7 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.mpp.plan.analyze;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.partition.DataPartition;
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
@@ -37,7 +38,6 @@ import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.mpp.common.MPPQueryContext;
import org.apache.iotdb.db.mpp.common.header.ColumnHeader;
-import org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant;
import org.apache.iotdb.db.mpp.common.header.DatasetHeader;
import org.apache.iotdb.db.mpp.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo;
@@ -127,6 +127,7 @@ import static com.google.common.base.Preconditions.checkState;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES;
+import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.COLUMN_DEVICE;
/** This visitor is used to analyze each type of Statement and returns the {@link Analysis}. */
public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> {
@@ -227,7 +228,12 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
List<Pair<Expression, String>> outputExpressions;
if (queryStatement.isAlignByDevice()) {
Map<String, Set<Expression>> deviceToTransformExpressions = new HashMap<>();
- Map<String, Set<Expression>> deviceToTransformExpressionsInHaving = new HashMap<>();
+ Expression deviceExpression =
+ new TimeSeriesOperand(new MeasurementPath(COLUMN_DEVICE, TSDataType.TEXT));
+ Set<Expression> transformInput = new LinkedHashSet<>();
+ transformInput.add(deviceExpression);
+ Set<Expression> transformOutput = new LinkedHashSet<>();
+ transformOutput.add(deviceExpression);
// all selected device
Set<PartialPath> deviceList = analyzeFrom(queryStatement, schemaTree);
@@ -239,49 +245,39 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
schemaTree,
deviceList,
deviceToTransformExpressions,
- deviceToMeasurementsMap);
+ deviceToMeasurementsMap,
+ transformInput);
+
+ transformOutput.addAll(
+ outputExpressions.stream().map(Pair::getLeft).collect(Collectors.toList()));
if (queryStatement.hasHaving()) {
- List<PartialPath> measurementNotExistDevices = new ArrayList<>();
- for (PartialPath device : deviceList) {
- try {
- deviceToTransformExpressionsInHaving.put(
- device.toString(),
- new HashSet<>(
- ExpressionAnalyzer.removeWildcardInFilterByDevice(
- queryStatement.getHavingCondition().getPredicate(),
- device,
- schemaTree,
- false)));
- } catch (SemanticException e) {
- if (e instanceof MeasurementNotExistException) {
- logger.warn(e.getMessage());
- measurementNotExistDevices.add(device);
- continue;
- }
- throw e;
- }
- }
- for (PartialPath measurementNotExistDevice : measurementNotExistDevices) {
- deviceList.remove(measurementNotExistDevice);
- deviceToTransformExpressions.remove(measurementNotExistDevice.getFullPath());
- }
+ Expression havingExpression =
+ analyzeHaving(
+ analysis,
+ schemaTree,
+ deviceList,
+ deviceToTransformExpressions,
+ deviceToMeasurementsMap,
+ transformInput);
+ analyzeExpression(analysis, havingExpression);
+
+ // used for planFilter after planDeviceView
+ analysis.setHavingExpression(havingExpression);
}
- Map<String, List<Integer>> deviceToMeasurementIndexesMap = new HashMap<>();
List<String> allMeasurements =
- outputExpressions.stream()
- .map(Pair::getLeft)
+ transformInput.stream()
.map(Expression::getExpressionString)
- .distinct()
.collect(Collectors.toList());
+
+ Map<String, List<Integer>> deviceToMeasurementIndexesMap = new HashMap<>();
for (String deviceName : deviceToMeasurementsMap.keySet()) {
List<String> measurementsUnderDevice =
new ArrayList<>(deviceToMeasurementsMap.get(deviceName));
List<Integer> indexes = new ArrayList<>();
for (String measurement : measurementsUnderDevice) {
- indexes.add(
- allMeasurements.indexOf(measurement) + 1); // add 1 to skip the device column
+ indexes.add(allMeasurements.indexOf(measurement));
}
deviceToMeasurementIndexesMap.put(deviceName, indexes);
}
@@ -294,12 +290,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Map<String, Set<Expression>> deviceToAggregationExpressions = new HashMap<>();
Map<String, Set<Expression>> deviceToAggregationTransformExpressions = new HashMap<>();
- Map<String, Expression> deviceToHavingExpression =
- new HashMap<>(); // store filter of every device
for (String deviceName : deviceToTransformExpressions.keySet()) {
Set<Expression> transformExpressions = deviceToTransformExpressions.get(deviceName);
- Set<Expression> transformExpressionsInHaving =
- deviceToTransformExpressionsInHaving.get(deviceName);
Set<Expression> aggregationExpressions = new LinkedHashSet<>();
Set<Expression> aggregationTransformExpressions = new LinkedHashSet<>();
@@ -310,24 +302,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
isHasRawDataInputAggregation =
analyzeAggregation(
transformExpressions, aggregationExpressions, aggregationTransformExpressions);
-
- if (queryStatement.hasHaving() && transformExpressionsInHaving != null) {
- List<Expression> aggregationExpressionsInHaving = new ArrayList<>();
- isHasRawDataInputAggregation |=
- analyzeAggregationInHaving(
- transformExpressionsInHaving,
- aggregationExpressionsInHaving,
- aggregationExpressions,
- aggregationTransformExpressions);
-
- Expression havingExpression;
- havingExpression =
- analyzeHavingSplitByDevice(
- transformExpressionsInHaving); // construct Filter from Having
- analyzeExpression(analysis, havingExpression);
- deviceToHavingExpression.put(deviceName, havingExpression);
- }
-
deviceToAggregationExpressions.put(deviceName, aggregationExpressions);
deviceToAggregationTransformExpressions.put(
deviceName, aggregationTransformExpressions);
@@ -344,23 +318,12 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
deviceToSourceExpressions.computeIfAbsent(deviceName, key -> new LinkedHashSet<>()),
isRawDataSource);
}
-
- if (queryStatement.hasHaving() && transformExpressionsInHaving != null) {
- for (Expression expression : transformExpressionsInHaving) {
- updateSource(
- expression,
- deviceToSourceExpressions.computeIfAbsent(
- deviceName, key -> new LinkedHashSet<>()),
- isRawDataSource);
- }
- }
deviceToIsRawDataSource.put(deviceName, isRawDataSource);
}
analysis.setDeviceToAggregationExpressions(deviceToAggregationExpressions);
analysis.setDeviceToAggregationTransformExpressions(
deviceToAggregationTransformExpressions);
analysis.setDeviceToIsRawDataSource(deviceToIsRawDataSource);
- analysis.setDeviceToHavingExpression(deviceToHavingExpression);
if (queryStatement.getWhereCondition() != null) {
Map<String, Expression> deviceToQueryFilter = new HashMap<>();
@@ -389,6 +352,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
analysis.setDeviceToQueryFilter(deviceToQueryFilter);
}
+ analysis.setTransformInput(transformInput);
+ analysis.setTransformOutput(transformOutput);
analysis.setDeviceToSourceExpressions(deviceToSourceExpressions);
analysis.setDeviceToTransformExpressions(deviceToTransformExpressions);
} else {
@@ -398,13 +363,20 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
.map(Pair::getLeft)
.collect(Collectors.toCollection(LinkedHashSet::new));
+ // cast functionName to lowercase in havingExpression
+ Expression havingExpression =
+ queryStatement.hasHaving()
+ ? ExpressionAnalyzer.removeAliasFromExpression(
+ queryStatement.getHavingCondition().getPredicate())
+ : null;
+
// get removeWildcard Expressions in Having
// used to analyzeAggregation in Having expression and updateSource
Set<Expression> transformExpressionsInHaving =
queryStatement.hasHaving()
? new HashSet<>(
ExpressionAnalyzer.removeWildcardInFilter(
- queryStatement.getHavingCondition().getPredicate(),
+ havingExpression,
queryStatement.getFromComponent().getPrefixPaths(),
schemaTree,
false))
@@ -439,9 +411,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
aggregationExpressions,
aggregationTransformExpressions);
- Expression havingExpression = // construct Filter from Having
+ havingExpression = // construct Filter from Having
analyzeHaving(
- queryStatement,
+ analysis,
analysis.getGroupByLevelExpressions(),
transformExpressionsInHaving,
aggregationExpressionsInHaving);
@@ -465,6 +437,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
if (queryStatement.hasHaving()) {
for (Expression expression : transformExpressionsInHaving) {
+ analyzeExpression(analysis, expression);
updateSource(expression, sourceExpressions, isRawDataSource);
}
}
@@ -514,7 +487,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
DataPartition dataPartition = fetchDataPartitionByDevices(deviceSet, schemaTree);
analysis.setDataPartitionInfo(dataPartition);
- } catch (StatementAnalyzeException e) {
+ } catch (StatementAnalyzeException | IllegalPathException e) {
logger.error("Meet error when analyzing the query statement: ", e);
throw new StatementAnalyzeException(
"Meet error when analyzing the query statement: " + e.getMessage());
@@ -595,7 +568,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
ISchemaTree schemaTree,
Set<PartialPath> deviceList,
Map<String, Set<Expression>> deviceToTransformExpressions,
- Map<String, Set<String>> deviceToMeasurementsMap) {
+ Map<String, Set<String>> deviceToMeasurementsMap,
+ Set<Expression> transformInput) {
+
QueryStatement queryStatement = (QueryStatement) analysis.getStatement();
List<Pair<Expression, String>> outputExpressions = new ArrayList<>();
ColumnPaginationController paginationController =
@@ -613,11 +588,22 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
for (PartialPath device : deviceList) {
List<Expression> transformExpressions =
ExpressionAnalyzer.concatDeviceAndRemoveWildcard(selectExpression, device, schemaTree);
+ if (queryStatement.isAggregationQuery()) {
+ // extract aggregation in transformExpressions as input of Transform node after DeviceView
+ // node
+ Set<Expression> aggregationExpressions = analyzeAggregation(transformExpressions);
+ for (Expression aggregationExpression : aggregationExpressions) {
+ Expression measurementExpression =
+ ExpressionAnalyzer.getMeasurementExpression(aggregationExpression);
+ transformInput.add(measurementExpression);
+ analyzeExpression(analysis, measurementExpression);
+ }
+ }
for (Expression transformExpression : transformExpressions) {
+ Expression measurementExpression =
+ ExpressionAnalyzer.getMeasurementExpression(transformExpression);
measurementToDeviceTransformExpressions
- .computeIfAbsent(
- ExpressionAnalyzer.getMeasurementExpression(transformExpression),
- key -> new LinkedHashMap<>())
+ .computeIfAbsent(measurementExpression, key -> new LinkedHashMap<>())
.put(
device.getFullPath(),
ExpressionAnalyzer.removeAliasFromExpression(transformExpression));
@@ -668,9 +654,27 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
deviceToTransformExpressions
.computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
.add(transformExpressionWithoutAlias);
- deviceToMeasurementsMap
- .computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
- .add(measurementExpressionWithoutAlias.getExpressionString());
+ if (queryStatement.isAggregationQuery()) {
+ deviceToMeasurementsMap
+ .computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
+ .addAll(
+ ExpressionAnalyzer.searchAggregationExpressions(
+ measurementExpressionWithoutAlias)
+ .stream()
+ .map(Expression::getExpressionString)
+ .collect(Collectors.toList()));
+ } else {
+ List<Expression> sourceExpressions =
+ ExpressionAnalyzer.searchSourceExpressions(
+ measurementExpressionWithoutAlias, true);
+ transformInput.addAll(sourceExpressions);
+ deviceToMeasurementsMap
+ .computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
+ .addAll(
+ sourceExpressions.stream()
+ .map(Expression::getExpressionString)
+ .collect(Collectors.toList()));
+ }
}
paginationController.consumeLimit();
} else {
@@ -682,6 +686,82 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return outputExpressions;
}
+ private Expression analyzeHaving(
+ Analysis analysis,
+ ISchemaTree schemaTree,
+ Set<PartialPath> deviceList,
+ Map<String, Set<Expression>> deviceToTransformExpressions,
+ Map<String, Set<String>> deviceToMeasurementsMap,
+ Set<Expression> transformInput) {
+ QueryStatement queryStatement = (QueryStatement) analysis.getStatement();
+ Expression havingExpression =
+ ExpressionAnalyzer.removeAliasFromExpression(
+ queryStatement.getHavingCondition().getPredicate());
+ Set<Expression> measurementsInHaving = new HashSet<>();
+
+ // measurement expression after removing wildcard
+ // use LinkedHashMap for order-preserving
+ Map<Expression, Map<String, Expression>> measurementToDeviceTransformExpressions =
+ new LinkedHashMap<>();
+ for (PartialPath device : deviceList) {
+ List<Expression> transformExpressionsInHaving =
+ ExpressionAnalyzer.concatDeviceAndRemoveWildcard(havingExpression, device, schemaTree);
+
+ measurementsInHaving.addAll(
+ transformExpressionsInHaving.stream()
+ .map(
+ transformExpression ->
+ ExpressionAnalyzer.getMeasurementExpression(transformExpression))
+ .collect(Collectors.toList()));
+
+ Set<Expression> aggregationExpressionsInHaving =
+ analyzeAggregation(transformExpressionsInHaving);
+
+ for (Expression aggregationExpressionInHaving : aggregationExpressionsInHaving) {
+ Expression measurementExpression =
+ ExpressionAnalyzer.getMeasurementExpression(aggregationExpressionInHaving);
+ transformInput.add(measurementExpression);
+ analyzeExpression(analysis, measurementExpression);
+ measurementToDeviceTransformExpressions
+ .computeIfAbsent(measurementExpression, key -> new LinkedHashMap<>())
+ .put(device.getFullPath(), aggregationExpressionInHaving);
+ }
+ }
+
+ for (Expression measurementExpression : measurementToDeviceTransformExpressions.keySet()) {
+
+ Map<String, Expression> deviceToTransformExpressionOfOneMeasurement =
+ measurementToDeviceTransformExpressions.get(measurementExpression);
+ deviceToTransformExpressionOfOneMeasurement
+ .values()
+ .forEach(expression -> analyzeExpression(analysis, expression));
+
+ // check whether the datatype of paths which has the same measurement name are
+ // consistent
+ // if not, throw a SemanticException
+ checkDataTypeConsistencyInAlignByDevice(
+ analysis, new ArrayList<>(deviceToTransformExpressionOfOneMeasurement.values()));
+
+ analyzeExpression(analysis, measurementExpression);
+
+ // add deviceToTransformExpressions
+ for (String deviceName : deviceToTransformExpressionOfOneMeasurement.keySet()) {
+ Expression transformExpression =
+ deviceToTransformExpressionOfOneMeasurement.get(deviceName);
+
+ deviceToTransformExpressions
+ .computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
+ .add(transformExpression);
+ deviceToMeasurementsMap
+ .computeIfAbsent(deviceName, key -> new LinkedHashSet<>())
+ .add(measurementExpression.getExpressionString());
+ }
+ }
+
+ return ExpressionUtils.constructQueryFilter(
+ measurementsInHaving.stream().distinct().collect(Collectors.toList()));
+ }
+
private Pair<Filter, Boolean> analyzeGlobalTimeFilter(QueryStatement queryStatement) {
Filter globalTimeFilter = null;
boolean hasValueFilter = false;
@@ -769,6 +849,17 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return isHasRawDataInputAggregation;
}
+ private Set<Expression> analyzeAggregation(List<Expression> inputExpressions) {
+ Set<Expression> aggregationExpressions = new HashSet<>();
+ for (Expression inputExpression : inputExpressions) {
+ for (Expression aggregationExpression :
+ ExpressionAnalyzer.searchAggregationExpressions(inputExpression)) {
+ aggregationExpressions.add(aggregationExpression);
+ }
+ }
+ return aggregationExpressions;
+ }
+
private Expression analyzeWhere(QueryStatement queryStatement, ISchemaTree schemaTree) {
List<Expression> rewrittenPredicates =
ExpressionAnalyzer.removeWildcardInFilter(
@@ -790,20 +881,23 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
private Expression analyzeHaving(
- QueryStatement queryStatement,
+ Analysis analysis,
Map<Expression, Set<Expression>> groupByLevelExpressions,
Set<Expression> transformExpressionsInHaving,
List<Expression> aggregationExpressionsInHaving) {
+ QueryStatement queryStatement = (QueryStatement) analysis.getStatement();
if (queryStatement.isGroupByLevel()) {
Map<Expression, Expression> rawPathToGroupedPathMapInHaving =
analyzeGroupByLevelInHaving(
- queryStatement, aggregationExpressionsInHaving, groupByLevelExpressions);
+ analysis, aggregationExpressionsInHaving, groupByLevelExpressions);
List<Expression> convertedPredicates = new ArrayList<>();
for (Expression expression : transformExpressionsInHaving) {
- convertedPredicates.add(
+ Expression convertedPredicate =
ExpressionAnalyzer.replaceRawPathWithGroupedPath(
- expression, rawPathToGroupedPathMapInHaving));
+ expression, rawPathToGroupedPathMapInHaving);
+ convertedPredicates.add(convertedPredicate);
+ analyzeExpression(analysis, expression);
}
return ExpressionUtils.constructQueryFilter(
convertedPredicates.stream().distinct().collect(Collectors.toList()));
@@ -813,21 +907,18 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
transformExpressionsInHaving.stream().distinct().collect(Collectors.toList()));
}
- private Expression analyzeHavingSplitByDevice(Set<Expression> transformExpressionsInHaving) {
- return ExpressionUtils.constructQueryFilter(
- transformExpressionsInHaving.stream().distinct().collect(Collectors.toList()));
- }
-
private Map<Expression, Expression> analyzeGroupByLevelInHaving(
- QueryStatement queryStatement,
+ Analysis analysis,
List<Expression> inputExpressions,
Map<Expression, Set<Expression>> groupByLevelExpressions) {
+ QueryStatement queryStatement = (QueryStatement) analysis.getStatement();
GroupByLevelController groupByLevelController =
new GroupByLevelController(queryStatement.getGroupByLevelComponent().getLevels());
for (Expression inputExpression : inputExpressions) {
groupByLevelController.control(false, inputExpression, null);
}
Map<Expression, Set<Expression>> groupedPathMap = groupByLevelController.getGroupedPathMap();
+ groupedPathMap.keySet().forEach(expression -> analyzeExpression(analysis, expression));
groupByLevelExpressions.putAll(groupedPathMap);
return groupByLevelController.getRawPathToGroupedPathMap();
}
@@ -910,8 +1001,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
queryStatement.isAggregationQuery() && !queryStatement.isGroupByTime();
List<ColumnHeader> columnHeaders = new ArrayList<>();
if (queryStatement.isAlignByDevice()) {
- columnHeaders.add(
- new ColumnHeader(ColumnHeaderConstant.COLUMN_DEVICE, TSDataType.TEXT, null));
+ columnHeaders.add(new ColumnHeader(COLUMN_DEVICE, TSDataType.TEXT, null));
}
columnHeaders.addAll(
outputExpressions.stream()
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index d8219177c3..8db8d3b6d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -483,7 +483,6 @@ public class ExpressionAnalyzer {
*
* @param prefixPaths prefix paths in the FROM clause
* @param schemaTree interface for querying schema information
- * @param typeProvider a map to record output symbols and their data types
* @return the expression list with full path and after binding schema
*/
public static List<Expression> removeWildcardInFilter(
@@ -1007,7 +1006,8 @@ public class ExpressionAnalyzer {
}
/**
- * Remove alias from expression. eg: root.sg.d1.status + 1 -> root.sg.d1.s2 + 1
+ * Remove alias from expression. eg: root.sg.d1.status + 1 -> root.sg.d1.s2 + 1, and reconstruct
+ * the name of FunctionExpression to lowercase.
*
* @return expression after removing alias
*/
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index c1b1e3de96..3b6c4e6436 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -82,7 +82,6 @@ import org.apache.iotdb.db.query.aggregation.AggregationType;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import org.apache.iotdb.tsfile.utils.Pair;
import com.google.common.base.Function;
import org.apache.commons.lang.Validate;
@@ -126,10 +125,13 @@ public class LogicalPlanBuilder {
private void updateTypeProvider(Collection<Expression> expressions) {
expressions.forEach(
- expression ->
+ expression -> {
+ if (!expression.getExpressionString().equals(COLUMN_DEVICE)) {
context
.getTypeProvider()
- .setType(expression.toString(), getPreAnalyzedType.apply(expression)));
+ .setType(expression.toString(), getPreAnalyzedType.apply(expression));
+ }
+ });
}
public LogicalPlanBuilder planRawDataSource(
@@ -456,15 +458,11 @@ public class LogicalPlanBuilder {
public LogicalPlanBuilder planDeviceView(
Map<String, PlanNode> deviceNameToSourceNodesMap,
- List<Pair<Expression, String>> outputExpressions,
+ Set<Expression> deviceViewOutput,
Map<String, List<Integer>> deviceToMeasurementIndexesMap,
Ordering mergeOrder) {
- List<String> outputColumnNames = new ArrayList<>();
- outputColumnNames.add(COLUMN_DEVICE);
- outputColumnNames.addAll(
- outputExpressions.stream()
- .map(pair -> pair.getLeft().toString())
- .collect(Collectors.toList()));
+ List<String> outputColumnNames =
+ deviceViewOutput.stream().map(Expression::getExpressionString).collect(Collectors.toList());
DeviceViewNode deviceViewNode =
new DeviceViewNode(
context.getQueryId().genPlanNodeId(),
@@ -474,6 +472,7 @@ public class LogicalPlanBuilder {
new SortItem(SortKey.TIME, mergeOrder))),
outputColumnNames,
deviceToMeasurementIndexesMap);
+
for (Map.Entry<String, PlanNode> entry : deviceNameToSourceNodesMap.entrySet()) {
String deviceName = entry.getKey();
PlanNode subPlan = entry.getValue();
@@ -481,7 +480,7 @@ public class LogicalPlanBuilder {
}
context.getTypeProvider().setType(COLUMN_DEVICE, TSDataType.TEXT);
- updateTypeProvider(outputExpressions.stream().map(Pair::getLeft).collect(Collectors.toList()));
+ updateTypeProvider(deviceViewOutput);
this.root = deviceViewNode;
return this;
@@ -647,7 +646,7 @@ public class LogicalPlanBuilder {
boolean isGroupByTime,
ZoneId zoneId,
Ordering scanOrder) {
- if (queryFilter == null) {
+ if (queryFilter == null || selectExpressions.isEmpty()) {
return this;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
index be44c14eee..8561ad6842 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
@@ -67,6 +67,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ActivateTemplate
import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ShowPathsUsingTemplateStatement;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -115,13 +116,13 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
analysis.getDeviceToSourceExpressions().get(deviceName),
analysis.getDeviceToAggregationExpressions().get(deviceName),
analysis.getDeviceToAggregationTransformExpressions().get(deviceName),
- analysis.getDeviceToTransformExpressions().get(deviceName),
+ analysis.hasValueFilter()
+ ? analysis.getDeviceToSourceExpressions().get(deviceName)
+ : Collections.emptySet(),
analysis.getDeviceToQueryFilter() != null
? analysis.getDeviceToQueryFilter().get(deviceName)
: null,
- analysis.getDeviceToHavingExpression() != null
- ? analysis.getDeviceToHavingExpression().get(deviceName)
- : null,
+ null,
analysis.getDeviceToMeasurementIndexesMap().get(deviceName),
context));
deviceToSubPlanMap.put(deviceName, subPlanBuilder.getRoot());
@@ -130,9 +131,23 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
planBuilder =
planBuilder.planDeviceView(
deviceToSubPlanMap,
- analysis.getOutputExpressions(),
+ analysis.getTransformInput(),
analysis.getDeviceToMeasurementIndexesMap(),
queryStatement.getResultTimeOrder());
+ if (queryStatement.hasHaving()) {
+ planBuilder.planFilterAndTransform(
+ analysis.getHavingExpression(),
+ analysis.getTransformOutput(),
+ queryStatement.isGroupByTime(),
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultTimeOrder());
+ } else {
+ planBuilder.planTransform(
+ analysis.getTransformOutput(),
+ queryStatement.isGroupByTime(),
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultTimeOrder());
+ }
} else {
planBuilder =
planBuilder.withNewRoot(
@@ -307,23 +322,6 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
aggregationExpressions,
aggregationTransformExpressions,
analysis.getGroupByLevelExpressions());
- if (queryStatement.isGroupByLevel()) {
- planBuilder = // plan Having with GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- analysis.getGroupByLevelExpressions().keySet(),
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- planBuilder = // plan Having without GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- }
} else {
planBuilder =
planBuilder.planAggregationSource(