You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/03/03 12:12:42 UTC
[iotdb] branch groupbylevelalias created (now a627d46)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch groupbylevelalias
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at a627d46 Fix the implement of alias with group by level
This branch includes the following new commits:
new a627d46 Fix the implement of alias with group by level
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Fix the implement of alias with group by level
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch groupbylevelalias
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a627d46ef0ca1d0a2984040325ceae8ca629bcbb
Author: Alima777 <wx...@gmail.com>
AuthorDate: Thu Mar 3 20:11:18 2022 +0800
Fix the implement of alias with group by level
---
.../aggregation/IoTDBAggregationByLevelIT.java | 160 +++++++++++++++++++++
.../iotdb/db/qp/physical/crud/AggregationPlan.java | 12 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 4 +-
.../iotdb/db/qp/utils/GroupByLevelController.java | 75 +++++++---
.../query/dataset/groupby/GroupByLevelDataSet.java | 3 +-
.../iotdb/db/query/expression/ResultColumn.java | 18 ++-
6 files changed, 246 insertions(+), 26 deletions(-)
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/aggregation/IoTDBAggregationByLevelIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/aggregation/IoTDBAggregationByLevelIT.java
index a2a3697..badd79a 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/aggregation/IoTDBAggregationByLevelIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/aggregation/IoTDBAggregationByLevelIT.java
@@ -312,6 +312,166 @@ public class IoTDBAggregationByLevelIT {
}
}
+ /**
+ * [root.sg.d1.temperature, root.sg.d2.temperature] with level = 1
+ *
+ * <p>Result is [root.sg.*.temperature]
+ */
+ @Test
+ public void groupByLevelWithAliasTest() throws Exception {
+ String[] retArray = new String[] {"5", "5", "5"};
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.d1, root.sg1.d2 GROUP BY level=1");
+
+ int cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("ct");
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ }
+
+ statement.execute("select count(temperature) as ct from root.sg1.* GROUP BY level=1");
+ cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("ct");
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ }
+
+ // root.sg1.d1.* -> [root.sg1.d1.status, root.sg1.d1.temperature] -> root.*.*.* -> ct
+ statement.execute("select count(*) as ct from root.sg1.d1 GROUP BY level=0");
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("ct");
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ }
+ }
+ }
+
+ /**
+ * [root.sg.d1.temperature, root.sg.d2.temperature] with level = 2
+ *
+ * <p>Result is [root.*.d1.temperature, root.*.d2.temperature]
+ */
+ @Test
+ public void groupByLevelWithAliasFailTest() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("select count(temperature) as ct from root.sg1.* GROUP BY level=2");
+ fail("No exception thrown");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("can only be matched with one"));
+ }
+ }
+
+ // Different from above at: root.sg1.*.temperature is just one ResultColumn
+ @Test
+ public void groupByLevelWithAliasFailTest2() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.d1, root.sg2.d2 GROUP BY level=2");
+ fail("No exception thrown");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("can only be matched with one"));
+ }
+ }
+
+ /** One Result Column with more than one alias. */
+ @Test
+ public void groupByLevelWithAliasFailTest3() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct, count(temperature) as ct2 from root.sg1.d1 GROUP BY level=2");
+ fail("No exception thrown");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("more than one alias"));
+ }
+ }
+
+ @Test
+ public void groupByLevelWithAliasWithTimeIntervalTest() throws Exception {
+ String[] retArray = new String[] {"0,0", "100,0", "200,2", "300,1", "400,0", "500,0"};
+ String[] retArray2 = new String[] {"0,0", "100,1", "200,2", "300,0", "400,0", "500,0"};
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.d1, root.sg1.d2 GROUP BY ([0, 600), 100ms), level=1");
+
+ int cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("Time") + "," + resultSet.getString("ct");
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ }
+
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.* GROUP BY ([0, 600), 100ms), level=1");
+
+ cnt = 0;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("Time") + "," + resultSet.getString("ct");
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ }
+
+ cnt = 0;
+ // root.sg1.d1.* -> [root.sg1.d1.status, root.sg1.d1.temperature] -> root.*.*.* -> ct
+ statement.execute(
+ "select count(*) as ct from root.sg1.d1 GROUP BY ([0, 600), 100ms), level=1");
+ try (ResultSet resultSet = statement.getResultSet()) {
+ while (resultSet.next()) {
+ String ans = resultSet.getString("Time") + "," + resultSet.getString("ct");
+ Assert.assertEquals(retArray2[cnt], ans);
+ cnt++;
+ }
+ }
+ }
+ }
+
+ /**
+ * [root.sg.d1.temperature, root.sg.d2.temperature] with level = 2
+ *
+ * <p>Result is [root.*.d1.temperature, root.*.d2.temperature]
+ */
+ @Test
+ public void groupByLevelWithAliasWithTimeIntervalFailTest() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.* GROUP BY ([0, 600), 100ms), level=2");
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("can only be matched with one"));
+ }
+ }
+
+ // Different from above at: root.sg1.*.temperature is just one ResultColumn
+ @Test
+ public void groupByLevelWithAliasWithTimeIntervalFailTest2() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "select count(temperature) as ct from root.sg1.d1, root.sg1.d2 GROUP BY ([0, 600), 100ms), level=2");
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("can only be matched with one"));
+ }
+ }
+
@Test
public void GroupByLevelSLimitTest() throws Exception {
String[] retArray = new String[] {"5,4", "4,6", "3"};
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/AggregationPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/AggregationPlan.java
index c81e12e..cb7a818 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/AggregationPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/AggregationPlan.java
@@ -67,7 +67,9 @@ public class AggregationPlan extends RawDataQueryPlan {
for (Map.Entry<String, AggregateResult> groupPathResult :
getGroupPathsResultMap().entrySet()) {
- respColumns.add(groupPathResult.getKey());
+ String resultColumnName = groupPathResult.getKey();
+ String aliasName = groupByLevelController.getAlias(resultColumnName);
+ respColumns.add(aliasName != null ? aliasName : resultColumnName);
columnsTypes.add(groupPathResult.getValue().getResultDataType().toString());
}
resp.setColumns(respColumns);
@@ -97,6 +99,10 @@ public class AggregationPlan extends RawDataQueryPlan {
return seriesTypes;
}
+ public GroupByLevelController getGroupByLevelController() {
+ return groupByLevelController;
+ }
+
@Override
public List<String> getAggregations() {
return aggregations;
@@ -163,7 +169,9 @@ public class AggregationPlan extends RawDataQueryPlan {
@Override
public String getColumnForReaderFromPath(PartialPath path, int pathIndex) {
- return resultColumns.get(pathIndex).getResultColumnName();
+ return isGroupByLevel()
+ ? resultColumns.get(pathIndex).getExpressionString()
+ : resultColumns.get(pathIndex).getResultColumnName();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 5e6a779..08b7a64 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -98,7 +98,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
List<PartialPath> prefixPaths = queryOperator.getFromComponent().getPrefixPaths();
List<ResultColumn> resultColumns = new ArrayList<>();
for (ResultColumn suffixColumn : queryOperator.getSelectComponent().getResultColumns()) {
- suffixColumn.concat(prefixPaths, resultColumns);
+ suffixColumn.concat(prefixPaths, resultColumns, queryOperator.isGroupByLevel());
}
queryOperator.getSelectComponent().setResultColumns(resultColumns);
}
@@ -120,7 +120,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
WildcardsRemover wildcardsRemover = new WildcardsRemover(queryOperator);
for (ResultColumn resultColumn : queryOperator.getSelectComponent().getResultColumns()) {
- resultColumn.removeWildcards(wildcardsRemover, resultColumns);
+ resultColumn.removeWildcards(wildcardsRemover, resultColumns, queryOperator.isGroupByLevel());
if (groupByLevelController != null) {
groupByLevelController.control(resultColumn, resultColumns);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/utils/GroupByLevelController.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/GroupByLevelController.java
index 8c09b2b..0c15d4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/utils/GroupByLevelController.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/GroupByLevelController.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.query.expression.ResultColumn;
import org.apache.iotdb.db.query.expression.unary.FunctionExpression;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -43,6 +44,9 @@ import java.util.Set;
*/
public class GroupByLevelController {
+ public static String ALIAS_ERROR_MESSAGE1 =
+ "alias '%s' can only be matched with one result column";
+ public static String ALIAS_ERROR_MESSAGE2 = "Result column %s with more than one alias[%s, %s]";
private final int seriesLimit;
private int seriesOffset;
Set<String> limitPaths;
@@ -51,6 +55,8 @@ public class GroupByLevelController {
int prevSize = 0;
/** count(root.sg.d1.s1) with level = 1 -> count(root.*.d1.s1) */
private Map<String, String> groupedPathMap;
+ /** count(root.*.d1.s1) -> alias */
+ private Map<String, String> aliasMap;
public GroupByLevelController(QueryOperator operator) {
this.seriesLimit = operator.getSpecialClauseComponent().getSeriesLimit();
@@ -65,31 +71,26 @@ public class GroupByLevelController {
return groupedPathMap.get(rawPath);
}
+ public String getAlias(String originName) {
+ return aliasMap != null && aliasMap.get(originName) != null ? aliasMap.get(originName) : null;
+ }
+
public void control(ResultColumn rawColumn, List<ResultColumn> resultColumns)
throws LogicalOptimizeException {
- Iterator<ResultColumn> iterator = resultColumns.iterator();
+ Set<Integer> countWildcardIterIndices = getCountStarIndices(rawColumn);
- // As one expression may have many aggregation results in the tree leaf, here we should traverse
- // all the successor expressions and record the count(*) indices
- Set<Integer> countWildcardIterIndices = new HashSet<>();
- int idx = 0;
- for (Iterator<Expression> it = rawColumn.getExpression().iterator(); it.hasNext(); ) {
- Expression expression = it.next();
- if (expression instanceof FunctionExpression
- && expression.isPlainAggregationFunctionExpression()
- && ((FunctionExpression) expression).isCountStar()) {
- countWildcardIterIndices.add(idx);
- }
- idx++;
- }
+ // `resultColumns` includes all result columns after removing wildcards, so we need to skip
+ // those we have processed
+ Iterator<ResultColumn> iterator = resultColumns.iterator();
for (int i = 0; i < prevSize; i++) {
iterator.next();
}
+
while (iterator.hasNext()) {
ResultColumn resultColumn = iterator.next();
Expression rootExpression = resultColumn.getExpression();
boolean hasAggregation = false;
- idx = 0;
+ int idx = 0;
for (Iterator<Expression> it = rootExpression.iterator(); it.hasNext(); ) {
Expression expression = it.next();
if (expression instanceof FunctionExpression
@@ -97,7 +98,7 @@ public class GroupByLevelController {
hasAggregation = true;
List<PartialPath> paths = ((FunctionExpression) expression).getPaths();
String functionName = ((FunctionExpression) expression).getFunctionName();
- boolean isCountStar = countWildcardIterIndices.contains(idx);
+ boolean isCountStar = countWildcardIterIndices.contains(idx++);
String groupedPath =
generatePartialPathByLevel(isCountStar, paths.get(0).getNodes(), levels);
String rawPath = String.format("%s(%s)", functionName, paths.get(0).getFullPath());
@@ -105,7 +106,9 @@ public class GroupByLevelController {
if (seriesLimit == 0 && seriesOffset == 0) {
groupedPathMap.put(rawPath, pathWithFunction);
+ checkAliasAndUpdateAliasMap(rawColumn, pathWithFunction);
} else {
+ // We cannot judge whether the path after grouping exists until we add it to set
if (seriesOffset > 0 && offsetPaths != null) {
offsetPaths.add(pathWithFunction);
if (offsetPaths.size() <= seriesOffset) {
@@ -121,13 +124,13 @@ public class GroupByLevelController {
limitPaths.remove(pathWithFunction);
} else {
groupedPathMap.put(rawPath, pathWithFunction);
+ checkAliasAndUpdateAliasMap(rawColumn, pathWithFunction);
}
} else {
iterator.remove();
}
}
}
- idx++;
}
if (!hasAggregation) {
throw new LogicalOptimizeException(rootExpression + " can't be used in group by level.");
@@ -136,6 +139,44 @@ public class GroupByLevelController {
prevSize = resultColumns.size();
}
+ // As one expression may have many aggregation results in the tree leaf, here we should traverse
+ // all the successor expressions and record the count(*) indices
+ private Set<Integer> getCountStarIndices(ResultColumn rawColumn) {
+ Set<Integer> countWildcardIterIndices = new HashSet<>();
+ int idx = 0;
+ for (Iterator<Expression> it = rawColumn.getExpression().iterator(); it.hasNext(); ) {
+ Expression expression = it.next();
+ if (expression instanceof FunctionExpression
+ && expression.isPlainAggregationFunctionExpression()
+ && ((FunctionExpression) expression).isCountStar()) {
+ countWildcardIterIndices.add(idx);
+ }
+ idx++;
+ }
+ return countWildcardIterIndices;
+ }
+
+ private void checkAliasAndUpdateAliasMap(ResultColumn rawColumn, String originName)
+ throws LogicalOptimizeException {
+ if (!rawColumn.hasAlias()) {
+ return;
+ } else if (aliasMap == null) {
+ aliasMap = new HashMap<>();
+ }
+ // If an alias is corresponding to more than one result column, throw an exception
+ if (aliasMap.get(originName) == null && aliasMap.containsValue(rawColumn.getAlias())) {
+ throw new LogicalOptimizeException(String.format(ALIAS_ERROR_MESSAGE1, rawColumn.getAlias()));
+ // If a result column is corresponding to more than one alias, throw an exception
+ } else if (aliasMap.get(originName) != null
+ && !aliasMap.get(originName).equals(rawColumn.getAlias())) {
+ throw new LogicalOptimizeException(
+ String.format(
+ ALIAS_ERROR_MESSAGE2, originName, aliasMap.get(originName), rawColumn.getAlias()));
+ } else {
+ aliasMap.put(originName, rawColumn.getAlias());
+ }
+ }
+
/**
* Transform an originalPath to a partial path that satisfies given level. Path nodes don't
* satisfy the given level will be replaced by "*" except the sensor level, e.g.
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByLevelDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByLevelDataSet.java
index f7f123e..6e9a5ab 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByLevelDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByLevelDataSet.java
@@ -71,7 +71,8 @@ public class GroupByLevelDataSet extends QueryDataSet {
if (paths.isEmpty()) {
for (Map.Entry<String, AggregateResult> entry : groupPathResultMap.entrySet()) {
try {
- this.paths.add(new PartialPath(entry.getKey()));
+ String alias = plan.getGroupByLevelController().getAlias(entry.getKey());
+ this.paths.add(new PartialPath(alias != null ? alias : entry.getKey()));
} catch (IllegalPathException e) {
logger.error("Query result IllegalPathException occurred: {}.", entry.getKey());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java b/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
index 51195d7..82bd140 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
@@ -88,12 +88,15 @@ public class ResultColumn {
/**
* @param prefixPaths prefix paths in the from clause
* @param resultColumns used to collect the result columns
+ * @param isGroupByLevel used to skip illegal alias judgement here. Because count(*) may be *
+ * unfolded to more than one expression, but it still can be aggregated together later
*/
- public void concat(List<PartialPath> prefixPaths, List<ResultColumn> resultColumns)
+ public void concat(
+ List<PartialPath> prefixPaths, List<ResultColumn> resultColumns, boolean isGroupByLevel)
throws LogicalOptimizeException {
List<Expression> resultExpressions = new ArrayList<>();
expression.concat(prefixPaths, resultExpressions);
- if (hasAlias() && 1 < resultExpressions.size()) {
+ if (!isGroupByLevel && hasAlias() && 1 < resultExpressions.size()) {
throw new LogicalOptimizeException(
String.format("alias '%s' can only be matched with one time series", alias));
}
@@ -106,12 +109,15 @@ public class ResultColumn {
* @param wildcardsRemover used to remove wildcards from {@code expression} and apply slimit &
* soffset control
* @param resultColumns used to collect the result columns
+ * @param isGroupByLevel used to skip illegal alias judgement here. Because count(*) may be
+ * unfolded to more than one expression, but it still can be aggregated together later
*/
- public void removeWildcards(WildcardsRemover wildcardsRemover, List<ResultColumn> resultColumns)
+ public void removeWildcards(
+ WildcardsRemover wildcardsRemover, List<ResultColumn> resultColumns, boolean isGroupByLevel)
throws LogicalOptimizeException {
List<Expression> resultExpressions = new ArrayList<>();
expression.removeWildcards(wildcardsRemover, resultExpressions);
- if (hasAlias() && 1 < resultExpressions.size()) {
+ if (!isGroupByLevel && hasAlias() && 1 < resultExpressions.size()) {
throw new LogicalOptimizeException(
String.format("alias '%s' can only be matched with one time series", alias));
}
@@ -145,6 +151,10 @@ public class ResultColumn {
return alias != null ? alias : expression.getExpressionString();
}
+ public String getExpressionString() {
+ return expression.getExpressionString();
+ }
+
public void setDataType(TSDataType dataType) {
this.dataType = dataType;
}