You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/10/08 11:33:59 UTC
[iotdb] branch master updated: [IOTDB-3818] Support nested expressions in GROUP BY LEVEL (#7510)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4b70531b3a [IOTDB-3818] Support nested expressions in GROUP BY LEVEL (#7510)
4b70531b3a is described below
commit 4b70531b3a5aadae667840c3dcfcb54ce875c220
Author: liuminghui233 <36...@users.noreply.github.com>
AuthorDate: Sat Oct 8 19:33:53 2022 +0800
[IOTDB-3818] Support nested expressions in GROUP BY LEVEL (#7510)
---
.../db/it/aligned/IoTDBGroupByLevelQueryIT.java | 64 +++++++++++++++++++++-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 9 ++-
.../mpp/plan/analyze/GroupByLevelController.java | 43 ++++++++++++---
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 12 ++--
4 files changed, 106 insertions(+), 22 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByLevelQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByLevelQueryIT.java
index 65300100e1..2a6693949e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByLevelQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBGroupByLevelQueryIT.java
@@ -37,7 +37,6 @@ import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
import static org.apache.iotdb.itbase.constant.TestConstant.NULL;
@RunWith(IoTDBTestRunner.class)
-@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBGroupByLevelQueryIT {
protected static boolean enableSeqSpaceCompaction;
@@ -98,6 +97,7 @@ public class IoTDBGroupByLevelQueryIT {
}
@Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
public void countFuncByLevelTest() {
// level = 1
double[][] retArray1 = new double[][] {{39, 20}};
@@ -134,6 +134,7 @@ public class IoTDBGroupByLevelQueryIT {
}
@Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
public void sumFuncByLevelTest() {
// level = 1
double[][] retArray1 = new double[][] {{131111, 510}};
@@ -167,6 +168,7 @@ public class IoTDBGroupByLevelQueryIT {
}
@Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
public void avgFuncByLevelTest() {
// level = 1
double[][] retArray1 = new double[][] {{2260.53448275862, 25.5}};
@@ -200,7 +202,8 @@ public class IoTDBGroupByLevelQueryIT {
}
@Test
- public void timeFuncGroupByLevelTest() throws ClassNotFoundException {
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void timeFuncGroupByLevelTest() {
double[][] retArray1 = new double[][] {{1, 40, 1, 30}};
String[] columnNames1 = {
"min_time(root.*.d1.s3)",
@@ -213,7 +216,8 @@ public class IoTDBGroupByLevelQueryIT {
}
@Test
- public void valueFuncGroupByLevelTest() throws ClassNotFoundException {
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void valueFuncGroupByLevelTest() {
double[][] retArray1 = new double[][] {{40, 230000, 30, 30}};
String[] columnNames1 = {
"last_value(root.*.d1.s3)",
@@ -226,4 +230,58 @@ public class IoTDBGroupByLevelQueryIT {
retArray1,
columnNames1);
}
+
+ @Test
+ @Category({ClusterIT.class})
+ public void nestedQueryTest1() {
+ // level = 1
+ double[][] retArray1 = new double[][] {{40.0, 21.0}};
+ String[] columnNames1 = {"count(root.sg1.*.s1 + 1) + 1", "count(root.sg2.*.s1 + 1) + 1"};
+ resultSetEqualTest(
+ "select count(s1 + 1) + 1 from root.*.* group by level=1", retArray1, columnNames1);
+
+ // level = 2
+ double[][] retArray2 = new double[][] {{41.0, 20.0}};
+ String[] columnNames2 = {"count(root.*.d1.s1 + 1) + 1", "count(root.*.d2.s1 + 1) + 1"};
+ resultSetEqualTest(
+ "select count(s1 + 1) + 1 from root.*.* group by level=2", retArray2, columnNames2);
+
+ // level = 3
+ double[][] retArray3 = new double[][] {{60.0}};
+ String[] columnNames3 = {"count(root.*.*.s1 + 1) + 1"};
+ resultSetEqualTest(
+ "select count(s1 + 1) + 1 from root.*.* group by level=3", retArray3, columnNames3);
+ }
+
+ @Test
+ @Category({ClusterIT.class})
+ public void nestedQueryTest2() {
+ // level = 1
+ double[][] retArray1 = new double[][] {{390423.0, 449.0, 390404.0, 430.0}};
+ String[] columnNames1 = {
+ "count(root.sg1.*.s1) + sum(root.sg1.*.s1)",
+ "count(root.sg1.*.s1) + sum(root.sg2.*.s1)",
+ "count(root.sg2.*.s1) + sum(root.sg1.*.s1)",
+ "count(root.sg2.*.s1) + sum(root.sg2.*.s1)"
+ };
+ resultSetEqualTest(
+ "select count(s1) + sum(s1) from root.*.* group by level=1", retArray1, columnNames1);
+
+ // level = 2
+ double[][] retArray2 = new double[][] {{390634.0, 240.0, 390613.0, 219.0}};
+ String[] columnNames2 = {
+ "count(root.*.d1.s1) + sum(root.*.d1.s1)",
+ "count(root.*.d1.s1) + sum(root.*.d2.s1)",
+ "count(root.*.d2.s1) + sum(root.*.d1.s1)",
+ "count(root.*.d2.s1) + sum(root.*.d2.s1)"
+ };
+ resultSetEqualTest(
+ "select count(s1) + sum(s1) from root.*.* group by level=2", retArray2, columnNames2);
+
+ // level = 3
+ double[][] retArray3 = new double[][] {{390853.0}};
+ String[] columnNames3 = {"count(root.*.*.s1) + sum(root.*.*.s1)"};
+ resultSetEqualTest(
+ "select count(s1) + sum(s1) from root.*.* group by level=3", retArray3, columnNames3);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index c5e0eb1179..89acdc9d59 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.db.exception.sql.MeasurementNotExistException;
import org.apache.iotdb.db.exception.sql.SemanticException;
-import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
@@ -52,7 +51,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -580,7 +578,8 @@ public class ExpressionAnalyzer {
}
public static Expression replaceRawPathWithGroupedPath(
- Expression expression, Map<NodeRef<PartialPath>, PartialPath> rawPathToGroupedPathMap) {
+ Expression expression,
+ GroupByLevelController.RawPathToGroupedPathMap rawPathToGroupedPathMap) {
if (expression instanceof TernaryExpression) {
Expression firstExpression =
replaceRawPathWithGroupedPath(
@@ -615,8 +614,8 @@ public class ExpressionAnalyzer {
}
return reconstructFunctionExpression((FunctionExpression) expression, childrenExpressions);
} else if (expression instanceof TimeSeriesOperand) {
- PartialPath groupedPath =
- rawPathToGroupedPathMap.get(NodeRef.of(((TimeSeriesOperand) expression).getPath()));
+ PartialPath rawPath = ((TimeSeriesOperand) expression).getPath();
+ PartialPath groupedPath = rawPathToGroupedPathMap.get(rawPath);
return reconstructTimeSeriesOperand(groupedPath);
} else if (expression instanceof TimestampOperand || expression instanceof ConstantOperand) {
return expression;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
index 9c33b41900..8641e3542e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
@@ -23,9 +23,9 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.sql.SemanticException;
-import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
+import org.apache.iotdb.tsfile.utils.Pair;
import java.util.ArrayList;
import java.util.HashMap;
@@ -36,6 +36,8 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import static com.google.common.base.Preconditions.checkState;
+
/**
* This class is used to control the row number of group by level query. For example, selected
* series[root.sg.d1.s1, root.sg.d2.s1, root.sg2.d1.s1], level = 1; the result rows will be
@@ -50,7 +52,7 @@ public class GroupByLevelController {
private final Map<Expression, Set<Expression>> groupedExpressionToRawExpressionsMap;
/** count(root.sg.d1.s1) with level = 1 -> { root.sg.d1.s1 : root.sg.*.s1 } */
- private final Map<NodeRef<PartialPath>, PartialPath> rawPathToGroupedPathMap;
+ private final RawPathToGroupedPathMap rawPathToGroupedPathMap;
/** count(root.*.d1.s1) -> alias */
private final Map<String, String> columnToAliasMap;
@@ -64,7 +66,7 @@ public class GroupByLevelController {
public GroupByLevelController(int[] levels) {
this.levels = levels;
this.groupedExpressionToRawExpressionsMap = new LinkedHashMap<>();
- this.rawPathToGroupedPathMap = new HashMap<>();
+ this.rawPathToGroupedPathMap = new RawPathToGroupedPathMap();
this.columnToAliasMap = new HashMap<>();
this.aliasToColumnMap = new HashMap<>();
}
@@ -75,14 +77,13 @@ public class GroupByLevelController {
public Expression control(boolean isCountStar, Expression expression, String alias) {
// update rawPathToGroupedPathMap
- Set<PartialPath> rawPaths =
+ List<PartialPath> rawPaths =
ExpressionAnalyzer.searchSourceExpressions(expression).stream()
.map(timeSeriesOperand -> ((TimeSeriesOperand) timeSeriesOperand).getPath())
- .collect(Collectors.toSet());
+ .collect(Collectors.toList());
for (PartialPath rawPath : rawPaths) {
- if (!rawPathToGroupedPathMap.containsKey(NodeRef.of(rawPath))) {
- rawPathToGroupedPathMap.put(
- NodeRef.of(rawPath), generatePartialPathByLevel(isCountStar, rawPath));
+ if (!rawPathToGroupedPathMap.containsKey(rawPath)) {
+ rawPathToGroupedPathMap.put(rawPath, generatePartialPathByLevel(isCountStar, rawPath));
}
}
@@ -178,4 +179,30 @@ public class GroupByLevelController {
public String getAlias(String columnName) {
return columnToAliasMap.get(columnName) != null ? columnToAliasMap.get(columnName) : null;
}
+
+ public static class RawPathToGroupedPathMap {
+
+ // key - a pair of raw path and its measurement alias
+ // value - grouped path
+ private final Map<Pair<PartialPath, String>, PartialPath> map = new HashMap<>();
+
+ public RawPathToGroupedPathMap() {
+ // do nothing
+ }
+
+ public boolean containsKey(PartialPath rawPath) {
+ return map.containsKey(new Pair<>(rawPath, rawPath.getMeasurementAlias()));
+ }
+
+ public void put(PartialPath rawPath, PartialPath groupedPath) {
+ map.put(new Pair<>(rawPath, rawPath.getMeasurementAlias()), groupedPath);
+ }
+
+ public PartialPath get(PartialPath rawPath) {
+ PartialPath groupedPath = map.get(new Pair<>(rawPath, rawPath.getMeasurementAlias()));
+ checkState(
+ groupedPath != null, "path '%s' is not analyzed in GroupByLevelController.", rawPath);
+ return groupedPath;
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index 7946ba6c5e..894c500112 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -421,13 +421,13 @@ public class LogicalPlanBuilder {
AggregationDescriptor aggregationDescriptor, TypeProvider typeProvider) {
List<AggregationType> splitAggregations =
SchemaUtils.splitPartialAggregation(aggregationDescriptor.getAggregationType());
- PartialPath path =
- ((TimeSeriesOperand) aggregationDescriptor.getInputExpressions().get(0)).getPath();
- for (AggregationType aggregationType : splitAggregations) {
- String functionName = aggregationType.toString().toLowerCase();
+ String inputExpressionStr = aggregationDescriptor.getInputExpressions().get(0).toString();
+ for (AggregationType aggregation : splitAggregations) {
+ String functionName = aggregation.toString().toLowerCase();
+ TSDataType aggregationType = SchemaUtils.getAggregationType(functionName);
typeProvider.setType(
- String.format("%s(%s)", functionName, path.getFullPath()),
- SchemaUtils.getSeriesTypeByPath(path, functionName));
+ String.format("%s(%s)", functionName, inputExpressionStr),
+ aggregationType == null ? typeProvider.getType(inputExpressionStr) : aggregationType);
}
}