You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2021/05/14 04:07:47 UTC
[iotdb] branch iotdb-1022-v2 updated: refactor concatSelect() in
ConcatPathOptimizer
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-1022-v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/iotdb-1022-v2 by this push:
new 18a8bb6 refactor concatSelect() in ConcatPathOptimizer
18a8bb6 is described below
commit 18a8bb614396f99deca61483b4a7953dd333fa54
Author: SteveYurongSu <st...@outlook.com>
AuthorDate: Fri May 14 12:06:59 2021 +0800
refactor concatSelect() in ConcatPathOptimizer
---
.../iotdb/db/qp/logical/crud/SelectOperator.java | 3 +
.../iotdb/db/qp/strategy/LogicalChecker.java | 52 ++-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 443 ++++++++-------------
.../PathNumberLimiter.java} | 16 +-
.../iotdb/db/query/expression/Expression.java | 5 +
.../iotdb/db/query/expression/ResultColumn.java | 17 +
.../query/expression/binary/BinaryExpression.java | 39 +-
.../query/expression/unary/FunctionExpression.java | 20 +
.../db/query/expression/unary/MinusExpression.java | 13 +
.../expression/unary/NumberLiteralOperand.java | 8 +
.../query/expression/unary/TimeSeriesOperand.java | 13 +
11 files changed, 328 insertions(+), 301 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectOperator.java
index 913ef74..434e44d 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectOperator.java
@@ -81,6 +81,9 @@ public final class SelectOperator extends Operator {
public void setResultColumns(List<ResultColumn> resultColumns) {
this.resultColumns = resultColumns;
+
+ pathsCache = null;
+ aggregationFunctionsCache = null;
}
public List<ResultColumn> getResultColumns() {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java
index d85c94c..b5a4202 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java
@@ -20,7 +20,10 @@
package org.apache.iotdb.db.qp.strategy;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
+import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.qp.logical.Operator;
+import org.apache.iotdb.db.qp.logical.crud.QueryOperator;
+import org.apache.iotdb.db.qp.logical.crud.SelectOperator;
public class LogicalChecker {
@@ -28,7 +31,54 @@ public class LogicalChecker {
* TODO: make check() an abstract method and call check() in this method or outside the
* LogicalChecker.
*/
- public static void check(Operator operator) throws LogicalOperatorException {}
+ public static void check(Operator operator) throws LogicalOperatorException {
+ if (operator instanceof QueryOperator) {
+ checkQueryOperator((QueryOperator) operator);
+ }
+ }
+
+ private static void checkQueryOperator(QueryOperator queryOperator)
+ throws LogicalOperatorException {
+ checkSelectOperator(queryOperator);
+ }
+
+ private static void checkSelectOperator(QueryOperator queryOperator)
+ throws LogicalOperatorException {
+ checkAggregation(queryOperator);
+ checkAlignByDevice(queryOperator);
+ }
+
+ private static void checkAggregation(QueryOperator queryOperator)
+ throws LogicalOperatorException {
+ SelectOperator selectOperator = queryOperator.getSelectOperator();
+ if (!selectOperator.getResultColumns().isEmpty()
+ && selectOperator.getPaths().size() != selectOperator.getAggregationFunctions().size()) {
+ throw new LogicalOperatorException(
+ "Common queries and aggregated queries are not allowed to appear at the same time");
+ }
+ }
+
+ private static void checkAlignByDevice(QueryOperator queryOperator)
+ throws LogicalOperatorException {
+ if (!queryOperator.isAlignByDevice()) {
+ return;
+ }
+
+ SelectOperator selectOperator = queryOperator.getSelectOperator();
+ if (selectOperator.hasTimeSeriesGeneratingFunction()) {
+ throw new LogicalOperatorException("ALIGN BY DEVICE clause is not supported in UDF queries.");
+ }
+
+ for (PartialPath path : selectOperator.getPaths()) {
+ String device = path.getDevice();
+ if (!device.isEmpty()) {
+ throw new LogicalOperatorException(
+ "The paths of the SELECT clause can only be single level. In other words, "
+ + "the paths of the SELECT clause can only be measurements or STAR, without DOT."
+ + " For more details please refer to the SQL document.");
+ }
+ }
+ }
private LogicalChecker() {}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index faee5e4..31e0744 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.qp.logical.crud.FunctionOperator;
import org.apache.iotdb.db.qp.logical.crud.QueryOperator;
import org.apache.iotdb.db.qp.logical.crud.SelectOperator;
import org.apache.iotdb.db.query.control.QueryResourceManager;
-import org.apache.iotdb.db.query.udf.core.context.UDFContext;
+import org.apache.iotdb.db.query.expression.ResultColumn;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -47,7 +47,8 @@ import java.util.Set;
/** concat paths in select and from clause. */
public class ConcatPathOptimizer implements ILogicalOptimizer {
- private static final Logger logger = LoggerFactory.getLogger(ConcatPathOptimizer.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(ConcatPathOptimizer.class);
+
private static final String WARNING_NO_SUFFIX_PATHS =
"failed to concat series paths because the given query operator didn't have suffix paths";
private static final String WARNING_NO_PREFIX_PATHS =
@@ -56,293 +57,46 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
@Override
public Operator transform(Operator operator, int fetchSize)
throws LogicalOptimizeException, PathNumOverLimitException {
- int maxDeduplicatedPathNum =
- QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize);
- if (operator instanceof QueryOperator && ((QueryOperator) operator).isLastQuery()) {
- // Dataset of last query actually has only three columns, so we shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
-
QueryOperator queryOperator = (QueryOperator) operator;
- FromOperator from = queryOperator.getFromOperator();
- List<PartialPath> prefixPaths;
- if (from == null) {
- logger.warn(WARNING_NO_PREFIX_PATHS);
- return operator;
- } else {
- prefixPaths = from.getPrefixPaths();
- if (prefixPaths.isEmpty()) {
- logger.warn(WARNING_NO_PREFIX_PATHS);
- return operator;
- }
- }
- SelectOperator select = queryOperator.getSelectOperator();
- List<PartialPath> initialSuffixPaths;
- if (select == null) {
- logger.warn(WARNING_NO_SUFFIX_PATHS);
- return operator;
- } else {
- initialSuffixPaths = select.getSuffixPaths();
- if (initialSuffixPaths.isEmpty()) {
- logger.warn(WARNING_NO_SUFFIX_PATHS);
- return operator;
- }
- }
-
- checkAggrOfSelectOperator(select);
-
- boolean isAlignByDevice = false;
- if (operator instanceof QueryOperator) {
- if (!((QueryOperator) operator).isAlignByDevice()
- || ((QueryOperator) operator).isLastQuery()) {
- // concat paths and remove stars
- int seriesLimit = ((QueryOperator) operator).getSeriesLimit();
- int seriesOffset = ((QueryOperator) operator).getSeriesOffset();
- concatSelect(
- prefixPaths,
- select,
- seriesLimit,
- seriesOffset,
- maxDeduplicatedPathNum,
- ((QueryOperator) operator).getIndexType() == null);
- } else {
- isAlignByDevice = true;
- if (((QueryOperator) operator).hasTimeSeriesGeneratingFunction()) {
- throw new LogicalOptimizeException(
- "ALIGN BY DEVICE clause is not supported in UDF queries.");
- }
- for (PartialPath path : initialSuffixPaths) {
- String device = path.getDevice();
- if (!device.isEmpty()) {
- throw new LogicalOptimizeException(
- "The paths of the SELECT clause can only be single level. In other words, "
- + "the paths of the SELECT clause can only be measurements or STAR, without DOT."
- + " For more details please refer to the SQL document.");
- }
- }
- // ALIGN_BY_DEVICE leaves the 1) concat, 2) remove star, 3) slimit tasks to the next phase,
- // i.e., PhysicalGenerator.transformQuery
- }
- }
-
- // concat filter
- FilterOperator filter = queryOperator.getFilterOperator();
- Set<PartialPath> filterPaths = new HashSet<>();
- if (filter == null) {
- return operator;
- }
- if (!isAlignByDevice) {
- queryOperator.setFilterOperator(concatFilter(prefixPaths, filter, filterPaths));
+ if (!optimizable(queryOperator)) {
+ return queryOperator;
}
- queryOperator.getFilterOperator().setPathSet(filterPaths);
- // GROUP_BY_DEVICE leaves the concatFilter to PhysicalGenerator to optimize filter without
- // prefix first
-
+ concatSelect(queryOperator);
+ removeWildcardsInSelectPaths(queryOperator);
+ concatFilter(queryOperator);
return queryOperator;
}
- private List<PartialPath> judgeSelectOperator(SelectOperator selectOperator)
- throws LogicalOptimizeException {
- List<PartialPath> suffixPaths;
- if (selectOperator == null) {
- throw new LogicalOptimizeException(WARNING_NO_SUFFIX_PATHS);
- } else {
- suffixPaths = selectOperator.getSuffixPaths();
- if (suffixPaths.isEmpty()) {
- throw new LogicalOptimizeException(WARNING_NO_SUFFIX_PATHS);
- }
- }
- return suffixPaths;
- }
-
- private void checkAggrOfSelectOperator(SelectOperator selectOperator)
- throws LogicalOptimizeException {
- if (!selectOperator.getAggregations().isEmpty()
- && selectOperator.getSuffixPaths().size() != selectOperator.getAggregations().size()) {
- throw new LogicalOptimizeException(
- "Common queries and aggregated queries are not allowed to appear at the same time");
- }
- }
-
- private void extendListSafely(List<String> source, int index, List<String> target) {
- if (source != null && !source.isEmpty()) {
- target.add(source.get(index));
+ private static boolean optimizable(QueryOperator queryOperator) {
+ FromOperator from = queryOperator.getFromOperator();
+ if (from == null || from.getPrefixPaths().isEmpty()) {
+ LOGGER.warn(WARNING_NO_PREFIX_PATHS);
+ return false;
}
- }
-
- /**
- * Extract paths from select&from cql, expand them into complete versions, and reassign them to
- * selectOperator's suffixPathList. Treat aggregations similarly.
- */
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- private void concatSelect(
- List<PartialPath> fromPaths,
- SelectOperator selectOperator,
- int limit,
- int offset,
- int maxDeduplicatedPathNum,
- boolean needRemoveStar)
- throws LogicalOptimizeException, PathNumOverLimitException {
- List<PartialPath> suffixPaths = judgeSelectOperator(selectOperator);
- List<PartialPath> afterConcatPaths = new ArrayList<>(); // null elements are for the UDFs
-
- List<String> originAggregations = selectOperator.getAggregations();
- List<String> afterConcatAggregations = new ArrayList<>(); // null elements are for the UDFs
-
- List<UDFContext> originUdfList = selectOperator.getUdfList();
- List<UDFContext> afterConcatUdfList = new ArrayList<>();
-
- for (int i = 0; i < suffixPaths.size(); i++) {
- // selectPath cannot start with ROOT, which is guaranteed by TSParser
- PartialPath selectPath = suffixPaths.get(i);
-
- if (selectPath == null) { // udf
- UDFContext originUdf = originUdfList.get(i);
- List<PartialPath> originUdfSuffixPaths = originUdf.getPaths();
-
- List<List<PartialPath>> afterConcatUdfPathsList = new ArrayList<>();
- for (PartialPath originUdfSuffixPath : originUdfSuffixPaths) {
- List<PartialPath> afterConcatUdfPaths = new ArrayList<>();
- for (PartialPath fromPath : fromPaths) {
- afterConcatUdfPaths.add(fromPath.concatPath(originUdfSuffixPath));
- }
- afterConcatUdfPathsList.add(afterConcatUdfPaths);
- }
- List<List<PartialPath>> extendedAfterConcatUdfPathsList = new ArrayList<>();
- cartesianProduct(
- afterConcatUdfPathsList, extendedAfterConcatUdfPathsList, 0, new ArrayList<>());
- for (List<PartialPath> afterConcatUdfPaths : extendedAfterConcatUdfPathsList) {
- afterConcatPaths.add(null);
- extendListSafely(originAggregations, i, afterConcatAggregations);
-
- afterConcatUdfList.add(
- new UDFContext(originUdf.getName(), originUdf.getAttributes(), afterConcatUdfPaths));
- }
- } else { // non-udf
- for (PartialPath fromPath : fromPaths) {
- PartialPath fullPath = fromPath.concatPath(selectPath);
- if (selectPath.isTsAliasExists()) {
- fullPath.setTsAlias(selectPath.getTsAlias());
- }
- afterConcatPaths.add(fullPath);
- extendListSafely(originAggregations, i, afterConcatAggregations);
-
- afterConcatUdfList.add(null);
- }
- }
+ SelectOperator select = queryOperator.getSelectOperator();
+ if (select == null || select.getResultColumns().isEmpty()) {
+ LOGGER.warn(WARNING_NO_SUFFIX_PATHS);
+ return false;
}
- if (needRemoveStar) {
- removeStarsInPath(
- afterConcatPaths,
- afterConcatAggregations,
- afterConcatUdfList,
- selectOperator,
- limit,
- offset,
- maxDeduplicatedPathNum);
- } else {
- selectOperator.setSuffixPathList(afterConcatPaths);
- }
+ return true;
}
- private FilterOperator concatFilter(
- List<PartialPath> fromPaths, FilterOperator operator, Set<PartialPath> filterPaths)
- throws LogicalOptimizeException {
- if (!operator.isLeaf()) {
- List<FilterOperator> newFilterList = new ArrayList<>();
- for (FilterOperator child : operator.getChildren()) {
- newFilterList.add(concatFilter(fromPaths, child, filterPaths));
- }
- operator.setChildren(newFilterList);
- return operator;
+ private void concatSelect(QueryOperator queryOperator) {
+ if (queryOperator.isAlignByDevice()) {
+ return;
}
- FunctionOperator functionOperator = (FunctionOperator) operator;
- PartialPath filterPath = functionOperator.getSinglePath();
- // do nothing in the cases of "where time > 5" or "where root.d1.s1 > 5"
- if (SQLConstant.isReservedPath(filterPath)
- || filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) {
- filterPaths.add(filterPath);
- return operator;
- }
- List<PartialPath> concatPaths = new ArrayList<>();
- fromPaths.forEach(fromPath -> concatPaths.add(fromPath.concatPath(filterPath)));
- List<PartialPath> noStarPaths = removeStarsInPathWithUnique(concatPaths);
- filterPaths.addAll(noStarPaths);
- if (noStarPaths.size() == 1) {
- // Transform "select s1 from root.car.* where s1 > 10" to
- // "select s1 from root.car.* where root.car.*.s1 > 10"
- functionOperator.setSinglePath(noStarPaths.get(0));
- return operator;
- } else {
- // Transform "select s1 from root.car.d1, root.car.d2 where s1 > 10" to
- // "select s1 from root.car.d1, root.car.d2 where root.car.d1.s1 > 10 and root.car.d2.s1 > 10"
- // Note that,
- // two fork tree has to be maintained while removing stars in paths for DnfFilterOptimizer
- // requirement.
- return constructBinaryFilterTreeWithAnd(noStarPaths, operator);
- }
- }
- private FilterOperator constructBinaryFilterTreeWithAnd(
- List<PartialPath> noStarPaths, FilterOperator operator) throws LogicalOptimizeException {
- FilterOperator filterBinaryTree = new FilterOperator(SQLConstant.KW_AND);
- FilterOperator currentNode = filterBinaryTree;
- for (int i = 0; i < noStarPaths.size(); i++) {
- if (i > 0 && i < noStarPaths.size() - 1) {
- FilterOperator newInnerNode = new FilterOperator(SQLConstant.KW_AND);
- currentNode.addChildOperator(newInnerNode);
- currentNode = newInnerNode;
- }
- try {
- currentNode.addChildOperator(
- new BasicFunctionOperator(
- operator.getTokenIntType(),
- noStarPaths.get(i),
- ((BasicFunctionOperator) operator).getValue()));
- } catch (SQLParserException e) {
- throw new LogicalOptimizeException(e.getMessage());
- }
+ List<PartialPath> prefixPaths = queryOperator.getFromOperator().getPrefixPaths();
+ List<ResultColumn> resultColumns = new ArrayList<>();
+ for (ResultColumn suffixColumn : queryOperator.getSelectOperator().getResultColumns()) {
+ suffixColumn.concat(prefixPaths, resultColumns);
}
- return filterBinaryTree;
+ queryOperator.getSelectOperator().setResultColumns(resultColumns);
}
- /**
- * replace "*" by actual paths.
- *
- * @param paths list of paths which may contain stars
- * @return a unique seriesPath list
- */
- private List<PartialPath> removeStarsInPathWithUnique(List<PartialPath> paths)
- throws LogicalOptimizeException {
- List<PartialPath> retPaths = new ArrayList<>();
- HashSet<PartialPath> pathSet = new HashSet<>();
- try {
- for (PartialPath path : paths) {
- List<PartialPath> all = removeWildcard(path, 0, 0).left;
- if (all.size() == 0) {
- throw new LogicalOptimizeException(
- String.format("Unknown time series %s in `where clause`", path));
- }
- for (PartialPath subPath : all) {
- if (!pathSet.contains(subPath)) {
- pathSet.add(subPath);
- retPaths.add(subPath);
- }
- }
- }
- } catch (MetadataException e) {
- throw new LogicalOptimizeException("error when remove star: " + e.getMessage());
- }
- return retPaths;
- }
-
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- private void removeStarsInPath(
+ private void removeWildcardsInSelectPaths(
List<PartialPath> afterConcatPaths,
List<String> afterConcatAggregations,
List<UDFContext> afterConcatUdfList,
@@ -351,6 +105,21 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
int finalOffset,
int maxDeduplicatedPathNum)
throws LogicalOptimizeException, PathNumOverLimitException {
+
+ int maxDeduplicatedPathNum =
+ QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize);
+ if (queryOperator.isLastQuery()) {
+ // Dataset of last query actually has only three columns, so we shouldn't limit the path num
+ // while constructing logical plan
+ // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we
+ // set it to Integer.MAX_VALUE - 1
+ maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
+ }
+
+ int seriesLimit = queryOperator.getSeriesLimit();
+ int seriesOffset = queryOperator.getSeriesOffset();
+ boolean needRemoveStar = queryOperator.getIndexType() == null;
+
int offset = finalOffset;
int limit =
finalLimit == 0 || maxDeduplicatedPathNum < finalLimit
@@ -450,11 +219,6 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
selectOperator.setUdfList(newUdfList);
}
- protected Pair<List<PartialPath>, Integer> removeWildcard(PartialPath path, int limit, int offset)
- throws MetadataException {
- return IoTDB.metaManager.getAllTimeseriesPathWithAlias(path, limit, offset);
- }
-
private void checkAndSetTsAlias(List<PartialPath> actualPaths, PartialPath originPath)
throws LogicalOptimizeException {
if (originPath.isTsAliasExists()) {
@@ -467,17 +231,126 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
}
}
- private static void cartesianProduct(
- List<List<PartialPath>> dimensionValue,
- List<List<PartialPath>> resultList,
- int layer,
- List<PartialPath> currentList) {
+ private void removeWildcardsInSelectPaths(QueryOperator queryOperator) {
+ if (queryOperator.getIndexType() != null) {
+ return;
+ }
+ }
+
+ private void concatFilter(QueryOperator queryOperator) throws LogicalOptimizeException {
+ FilterOperator filter = queryOperator.getFilterOperator();
+ if (filter == null) {
+ return;
+ }
+
+ if (queryOperator.isAlignByDevice()) {
+ // GROUP_BY_DEVICE leaves the concatFilter to PhysicalGenerator to optimize filter without
+ // prefix first
+ queryOperator.getFilterOperator().setPathSet(new HashSet<>());
+ } else {
+ queryOperator.setFilterOperator(
+ concatFilter(queryOperator.getFromOperator().getPrefixPaths(), filter, new HashSet<>()));
+ }
+ }
+
+ private FilterOperator concatFilter(
+ List<PartialPath> fromPaths, FilterOperator operator, Set<PartialPath> filterPaths)
+ throws LogicalOptimizeException {
+ if (!operator.isLeaf()) {
+ List<FilterOperator> newFilterList = new ArrayList<>();
+ for (FilterOperator child : operator.getChildren()) {
+ newFilterList.add(concatFilter(fromPaths, child, filterPaths));
+ }
+ operator.setChildren(newFilterList);
+ return operator;
+ }
+ FunctionOperator functionOperator = (FunctionOperator) operator;
+ PartialPath filterPath = functionOperator.getSinglePath();
+ // do nothing in the cases of "where time > 5" or "where root.d1.s1 > 5"
+ if (SQLConstant.isReservedPath(filterPath)
+ || filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) {
+ filterPaths.add(filterPath);
+ return operator;
+ }
+ List<PartialPath> concatPaths = new ArrayList<>();
+ fromPaths.forEach(fromPath -> concatPaths.add(fromPath.concatPath(filterPath)));
+ List<PartialPath> noStarPaths = removeWildcardsInConcatPaths(concatPaths);
+ filterPaths.addAll(noStarPaths);
+ if (noStarPaths.size() == 1) {
+ // Transform "select s1 from root.car.* where s1 > 10" to
+ // "select s1 from root.car.* where root.car.*.s1 > 10"
+ functionOperator.setSinglePath(noStarPaths.get(0));
+ return operator;
+ } else {
+ // Transform "select s1 from root.car.d1, root.car.d2 where s1 > 10" to
+ // "select s1 from root.car.d1, root.car.d2 where root.car.d1.s1 > 10 and root.car.d2.s1 > 10"
+ // Note that,
+ // two fork tree has to be maintained while removing stars in paths for DnfFilterOptimizer
+ // requirement.
+ return constructBinaryFilterTreeWithAnd(noStarPaths, operator);
+ }
+ }
+
+ private FilterOperator constructBinaryFilterTreeWithAnd(
+ List<PartialPath> noStarPaths, FilterOperator operator) throws LogicalOptimizeException {
+ FilterOperator filterBinaryTree = new FilterOperator(SQLConstant.KW_AND);
+ FilterOperator currentNode = filterBinaryTree;
+ for (int i = 0; i < noStarPaths.size(); i++) {
+ if (i > 0 && i < noStarPaths.size() - 1) {
+ FilterOperator newInnerNode = new FilterOperator(SQLConstant.KW_AND);
+ currentNode.addChildOperator(newInnerNode);
+ currentNode = newInnerNode;
+ }
+ try {
+ currentNode.addChildOperator(
+ new BasicFunctionOperator(
+ operator.getTokenIntType(),
+ noStarPaths.get(i),
+ ((BasicFunctionOperator) operator).getValue()));
+ } catch (SQLParserException e) {
+ throw new LogicalOptimizeException(e.getMessage());
+ }
+ }
+ return filterBinaryTree;
+ }
+
+ private List<PartialPath> removeWildcardsInConcatPaths(List<PartialPath> paths)
+ throws LogicalOptimizeException {
+ List<PartialPath> retPaths = new ArrayList<>();
+ HashSet<PartialPath> pathSet = new HashSet<>();
+ try {
+ for (PartialPath path : paths) {
+ List<PartialPath> all = removeWildcard(path, 0, 0).left;
+ if (all.size() == 0) {
+ throw new LogicalOptimizeException(
+ String.format("Unknown time series %s in `where clause`", path));
+ }
+ for (PartialPath subPath : all) {
+ if (!pathSet.contains(subPath)) {
+ pathSet.add(subPath);
+ retPaths.add(subPath);
+ }
+ }
+ }
+ } catch (MetadataException e) {
+ throw new LogicalOptimizeException("error when remove star: " + e.getMessage());
+ }
+ return retPaths;
+ }
+
+ protected Pair<List<PartialPath>, Integer> removeWildcard(PartialPath path, int limit, int offset)
+ throws MetadataException {
+ return IoTDB.metaManager.getAllTimeseriesPathWithAlias(path, limit, offset);
+ }
+
+ public static <T> void cartesianProduct(
+ List<List<T>> dimensionValue, List<List<T>> resultList, int layer, List<T> currentList) {
if (layer < dimensionValue.size() - 1) {
if (dimensionValue.get(layer).isEmpty()) {
cartesianProduct(dimensionValue, resultList, layer + 1, currentList);
} else {
for (int i = 0; i < dimensionValue.get(layer).size(); i++) {
- List<PartialPath> list = new ArrayList<>(currentList);
+ List<T> list = new ArrayList<>(currentList);
list.add(dimensionValue.get(layer).get(i));
cartesianProduct(dimensionValue, resultList, layer + 1, list);
}
@@ -487,7 +360,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
resultList.add(currentList);
} else {
for (int i = 0; i < dimensionValue.get(layer).size(); i++) {
- List<PartialPath> list = new ArrayList<>(currentList);
+ List<T> list = new ArrayList<>(currentList);
list.add(dimensionValue.get(layer).get(i));
resultList.add(list);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/PathNumberLimiter.java
similarity index 65%
copy from server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java
copy to server/src/main/java/org/apache/iotdb/db/qp/utils/PathNumberLimiter.java
index d85c94c..d312fb8 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalChecker.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/PathNumberLimiter.java
@@ -17,18 +17,6 @@
* under the License.
*/
-package org.apache.iotdb.db.qp.strategy;
+package org.apache.iotdb.db.qp.utils;
-import org.apache.iotdb.db.exception.query.LogicalOperatorException;
-import org.apache.iotdb.db.qp.logical.Operator;
-
-public class LogicalChecker {
-
- /**
- * TODO: make check() an abstract method and call check() in this method or outside the
- * LogicalChecker.
- */
- public static void check(Operator operator) throws LogicalOperatorException {}
-
- private LogicalChecker() {}
-}
+public class PathNumberLimiter {}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java b/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
index dc2700e..70f5a85 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
@@ -20,8 +20,11 @@
package org.apache.iotdb.db.query.expression;
import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.List;
+
public abstract class Expression {
protected boolean isAggregationFunctionExpression = false;
@@ -36,4 +39,6 @@ public abstract class Expression {
}
public abstract TSDataType dataType() throws MetadataException;
+
+ public abstract void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java b/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
index 482d3b5..3289c53 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/ResultColumn.java
@@ -19,6 +19,11 @@
package org.apache.iotdb.db.query.expression;
+import org.apache.iotdb.db.metadata.PartialPath;
+
+import java.util.ArrayList;
+import java.util.List;
+
public class ResultColumn {
private final Expression expression;
@@ -34,10 +39,22 @@ public class ResultColumn {
alias = null;
}
+ public void concat(List<PartialPath> prefixPaths, List<ResultColumn> resultColumns) {
+ List<Expression> resultExpressions = new ArrayList<>();
+ expression.concat(prefixPaths, resultExpressions);
+ for (Expression resultExpression : resultExpressions) {
+ resultColumns.add(new ResultColumn(resultExpression, alias));
+ }
+ }
+
public Expression getExpression() {
return expression;
}
+ public String getAlias() {
+ return alias;
+ }
+
public String getResultColumnName() {
return alias != null ? alias : expression.toString();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/binary/BinaryExpression.java b/server/src/main/java/org/apache/iotdb/db/query/expression/binary/BinaryExpression.java
index 741e3f5..97047e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/binary/BinaryExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/binary/BinaryExpression.java
@@ -19,9 +19,13 @@
package org.apache.iotdb.db.query.expression.binary;
+import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.ArrayList;
+import java.util.List;
+
public abstract class BinaryExpression extends Expression {
protected BinaryExpression(Expression leftExpression, Expression rightExpression) {
@@ -38,11 +42,44 @@ public abstract class BinaryExpression extends Expression {
* <p>TODO: This is just a simple implementation and should be optimized later.
*/
@Override
- public TSDataType dataType() {
+ public final TSDataType dataType() {
return TSDataType.DOUBLE;
}
@Override
+ public final void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
+ List<Expression> leftExpressions = new ArrayList<>();
+ leftExpression.concat(prefixPaths, leftExpressions);
+
+ List<Expression> rightExpressions = new ArrayList<>();
+ rightExpression.concat(prefixPaths, rightExpressions);
+
+ for (Expression le : leftExpressions) {
+ for (Expression re : resultExpressions) {
+ switch (operator()) {
+ case "+":
+ resultExpressions.add(new AdditionExpression(le, re));
+ break;
+ case "-":
+ resultExpressions.add(new SubtractionExpression(le, re));
+ break;
+ case "*":
+ resultExpressions.add(new MultiplicationExpression(le, re));
+ break;
+ case "/":
+ resultExpressions.add(new DivisionExpression(le, re));
+ break;
+ case "%":
+ resultExpressions.add(new ModuloExpression(le, re));
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+ }
+
+ @Override
public final String toString() {
return String.format(
"%s %s %s", leftExpression.toString(), operator(), rightExpression.toString());
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/FunctionExpression.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/FunctionExpression.java
index ab3bf3e..b5b4950 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/FunctionExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/FunctionExpression.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.query.expression.unary;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.qp.constant.SQLConstant;
+import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.tsfile.exception.NotImplementedException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -98,6 +99,25 @@ public class FunctionExpression extends Expression {
throw new NotImplementedException();
}
+ @Override
+ public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
+ List<List<Expression>> resultExpressionsForRecursionList = new ArrayList<>();
+
+ for (Expression suffixExpression : expressions) {
+ List<Expression> resultExpressionsForRecursion = new ArrayList<>();
+ suffixExpression.concat(prefixPaths, resultExpressionsForRecursion);
+ resultExpressionsForRecursionList.add(resultExpressionsForRecursion);
+ }
+
+ List<List<Expression>> functionExpressions = new ArrayList<>();
+ ConcatPathOptimizer.cartesianProduct(
+ resultExpressionsForRecursionList, functionExpressions, 0, new ArrayList<>());
+ for (List<Expression> functionExpression : functionExpressions) {
+ resultExpressions.add(
+ new FunctionExpression(functionName, functionAttributes, functionExpression));
+ }
+ }
+
public List<TSDataType> getDataTypes() throws MetadataException {
if (dataTypes == null) {
dataTypes = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/MinusExpression.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/MinusExpression.java
index 6a3145b..3dd0a8f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/MinusExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/MinusExpression.java
@@ -20,9 +20,13 @@
package org.apache.iotdb.db.query.expression.unary;
import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.ArrayList;
+import java.util.List;
+
public class MinusExpression extends Expression {
protected Expression expression;
@@ -41,6 +45,15 @@ public class MinusExpression extends Expression {
}
@Override
+ public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
+ List<Expression> resultExpressionsForRecursion = new ArrayList<>();
+ expression.concat(prefixPaths, resultExpressionsForRecursion);
+ for (Expression resultExpression : resultExpressionsForRecursion) {
+ resultExpressions.add(new MinusExpression(resultExpression));
+ }
+ }
+
+ @Override
public String toString() {
return "-" + expression.toString();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/NumberLiteralOperand.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/NumberLiteralOperand.java
index 7eaf306..e0868ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/NumberLiteralOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/NumberLiteralOperand.java
@@ -19,9 +19,12 @@
package org.apache.iotdb.db.query.expression.unary;
+import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.List;
+
public class NumberLiteralOperand extends Expression {
protected double literal;
@@ -36,6 +39,11 @@ public class NumberLiteralOperand extends Expression {
}
@Override
+ public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public String toString() {
return String.valueOf(literal);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
index bc4e635..429e509 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
@@ -25,6 +25,8 @@ import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.List;
+
public class TimeSeriesOperand extends Expression {
protected PartialPath path;
@@ -51,6 +53,17 @@ public class TimeSeriesOperand extends Expression {
}
@Override
+ public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
+ for (PartialPath prefixPath : prefixPaths) {
+ PartialPath fullPath = prefixPath.concatPath(path);
+ if (path.isTsAliasExists()) {
+ fullPath.setTsAlias(path.getTsAlias());
+ }
+ resultExpressions.add(new TimeSeriesOperand(fullPath));
+ }
+ }
+
+ @Override
public String toString() {
return path.toString();
}