You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/08/13 14:26:33 UTC

[iotdb] branch beyyes/count_time_fe_temp created (now 4008f7d9f02)

This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a change to branch beyyes/count_time_fe_temp
in repository https://gitbox.apache.org/repos/asf/iotdb.git


      at 4008f7d9f02 reimpl according review

This branch includes the following new commits:

     new 4008f7d9f02 reimpl according review

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[iotdb] 01/01: reimpl according review

Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a commit to branch beyyes/count_time_fe_temp
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 4008f7d9f0220112b2cbbfdcc982bfdeea855f82
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Sun Aug 13 22:25:36 2023 +0800

    reimpl according review
---
 .../db/queryengine/plan/analyze/Analysis.java      |   3 +-
 .../queryengine/plan/analyze/AnalyzeVisitor.java   | 125 ++++++++++-----------
 .../visitor/CountTimeAggregationAmountVisitor.java |  46 ++++++++
 .../plan/statement/crud/QueryStatement.java        |  34 +++---
 4 files changed, 123 insertions(+), 85 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
index 05329d849e2..105cc232bb0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
@@ -50,6 +50,7 @@ import org.apache.iotdb.tsfile.utils.Pair;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -93,7 +94,7 @@ public class Analysis {
   private Set<Expression> sourceExpressions;
 
   // input expressions of aggregations to be calculated
-  private Set<Expression> sourceTransformExpressions;
+  private Set<Expression> sourceTransformExpressions = new HashSet<>();
 
   private Expression whereExpression;
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 342dcdffa2f..ab51437ee9a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -170,6 +170,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -303,6 +304,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
         }
 
         analyzeGroupBy(analysis, queryStatement, schemaTree);
+
         analyzeHaving(analysis, queryStatement, schemaTree);
         analyzeOrderBy(analysis, queryStatement, schemaTree);
 
@@ -310,18 +312,28 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
         analyzeGroupByTag(analysis, queryStatement, outputExpressions);
 
         Set<Expression> selectExpressions = new LinkedHashSet<>();
+        // duplicatedAggExpressions is used for count_time aggregation,
+        // because both count_time(s1) and count_time(s2) is transformed to count_time(Time),
+        // but we need put the countTimeSeries to sourceTransformedExpression
+        List<Expression> duplicatedAggExpressions = new ArrayList<>();
         if (queryStatement.isOutputEndTime()) {
           selectExpressions.add(END_TIME_EXPRESSION);
         }
         for (Pair<Expression, String> outputExpressionAndAlias : outputExpressions) {
-          selectExpressions.add(outputExpressionAndAlias.left);
+          Expression outputExpression = outputExpressionAndAlias.left;
+          if (queryStatement.isAggregationQuery() && selectExpressions.contains(outputExpression)) {
+            duplicatedAggExpressions.add(outputExpression);
+            continue;
+          }
+
+          selectExpressions.add(outputExpression);
         }
         analysis.setSelectExpressions(selectExpressions);
 
         analyzeAggregation(analysis, queryStatement);
 
         analyzeWhere(analysis, queryStatement, schemaTree);
-        analyzeSourceTransform(analysis, queryStatement);
+        analyzeSourceTransform(analysis, queryStatement, duplicatedAggExpressions);
 
         analyzeSource(analysis, queryStatement);
 
@@ -541,12 +553,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
           } else {
             Expression normalizedExpression = normalizeExpression(resultExpression);
             analyzeExpressionType(analysis, normalizedExpression);
-
-            if (queryStatement.isAggregationQuery()
-                && normalizedExpression.getOutputSymbol().contains(COUNT_TIME)) {
-              analyzeSelectOfCountTimeAggregation(analysis, resultExpression);
-            }
-
             checkAliasUniqueness(resultColumn.getAlias(), aliasSet);
             outputExpressions.add(
                 new Pair<>(
@@ -568,34 +574,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
     return outputExpressionMap;
   }
 
-  /**
-   * process for count_time aggregation particularly, resultExpression can be `count_time(s1)`,
-   * `count_time(*)`, count_time(s1+s2), sum(s1)/count_time(*), etc.
-   */
-  private void analyzeSelectOfCountTimeAggregation(Analysis analysis, Expression resultExpression) {
-    Set<Expression> sourceTransformExpressions = analysis.getSourceTransformExpressions();
-    if (sourceTransformExpressions == null) {
-      sourceTransformExpressions = new HashSet<>();
-      analysis.setSourceTransformExpressions(sourceTransformExpressions);
-    }
-
-    List<Expression> allAggregationExpressions = searchAggregationExpressions(resultExpression);
-    for (Expression subAggregationExpression : allAggregationExpressions) {
-      if (subAggregationExpression instanceof FunctionExpression
-          && COUNT_TIME.equalsIgnoreCase(
-              ((FunctionExpression) subAggregationExpression).getFunctionName())) {
-
-        // variable countTimeExpressions of FunctionExpression stores all the expressions that
-        // count_time aggregation used
-        for (Expression countTimeSourceExpression :
-            ((FunctionExpression) subAggregationExpression).getCountTimeExpressions()) {
-          analyzeExpressionType(analysis, countTimeSourceExpression);
-          sourceTransformExpressions.add(countTimeSourceExpression);
-        }
-      }
-    }
-  }
-
   private Set<PartialPath> analyzeFrom(QueryStatement queryStatement, ISchemaTree schemaTree) {
     // device path patterns in FROM clause
     List<PartialPath> devicePatternList = queryStatement.getFromComponent().getPrefixPaths();
@@ -822,6 +800,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
         ExpressionUtils.constructQueryFilter(
             conJunctions.stream().distinct().collect(Collectors.toList()));
     havingExpression = normalizeExpression(havingExpression);
+
     TSDataType outputType = analyzeExpressionType(analysis, havingExpression);
     if (outputType != TSDataType.BOOLEAN) {
       throw new SemanticException(
@@ -829,6 +808,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
               "The output type of the expression in HAVING clause should be BOOLEAN, actual data type: %s.",
               outputType));
     }
+
     analysis.setHavingExpression(havingExpression);
   }
 
@@ -1123,21 +1103,22 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
               .flatMap(Set::stream)
               .collect(Collectors.toSet());
       analysis.setAggregationExpressions(aggregationExpressions);
-    } else {
-      Set<Expression> aggregationExpressions = new HashSet<>();
-      for (Expression expression : analysis.getSelectExpressions()) {
+      return;
+    }
+
+    Set<Expression> aggregationExpressions = new HashSet<>();
+    for (Expression expression : analysis.getSelectExpressions()) {
+      aggregationExpressions.addAll(searchAggregationExpressions(expression));
+    }
+    if (queryStatement.hasHaving()) {
+      aggregationExpressions.addAll(searchAggregationExpressions(analysis.getHavingExpression()));
+    }
+    if (queryStatement.hasOrderByExpression()) {
+      for (Expression expression : analysis.getOrderByExpressions()) {
         aggregationExpressions.addAll(searchAggregationExpressions(expression));
       }
-      if (queryStatement.hasHaving()) {
-        aggregationExpressions.addAll(searchAggregationExpressions(analysis.getHavingExpression()));
-      }
-      if (queryStatement.hasOrderByExpression()) {
-        for (Expression expression : analysis.getOrderByExpressions()) {
-          aggregationExpressions.addAll(searchAggregationExpressions(expression));
-        }
-      }
-      analysis.setAggregationExpressions(aggregationExpressions);
     }
+    analysis.setAggregationExpressions(aggregationExpressions);
   }
 
   private void analyzeDeviceToSourceTransform(Analysis analysis, QueryStatement queryStatement) {
@@ -1215,36 +1196,44 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
     }
   }
 
-  private void analyzeSourceTransform(Analysis analysis, QueryStatement queryStatement) {
+  private void analyzeSourceTransform(
+      Analysis analysis, QueryStatement queryStatement, List<Expression> duplicatedAggExpressions) {
     Set<Expression> sourceTransformExpressions = analysis.getSourceTransformExpressions();
-    if (sourceTransformExpressions == null) {
-      sourceTransformExpressions = new HashSet<>();
-      analysis.setSourceTransformExpressions(sourceTransformExpressions);
-    }
 
     if (queryStatement.isAggregationQuery()) {
-      for (Expression aggregationExpression : analysis.getAggregationExpressions()) {
-        // do not put "TimestampOperand" into sourceTransformExpression
-        if (aggregationExpression instanceof FunctionExpression
-            && COUNT_TIME.equalsIgnoreCase(
-                ((FunctionExpression) aggregationExpression).getFunctionName())) {
-          continue;
-        }
-
-        // for AggregationExpression, only the first Expression of input need to transform
-        sourceTransformExpressions.add(aggregationExpression.getExpressions().get(0));
+      putAggExpressionToSourceTransform(analysis.getAggregationExpressions(), analysis);
+      for (Expression duplicateExpression : duplicatedAggExpressions) {
+        putAggExpressionToSourceTransform(
+            searchAggregationExpressions(duplicateExpression), analysis);
       }
 
       if (queryStatement.hasGroupByExpression()) {
         sourceTransformExpressions.add(analysis.getGroupByExpression());
       }
-
-      return;
+    } else {
+      sourceTransformExpressions.addAll(analysis.getSelectExpressions());
+      if (queryStatement.hasOrderByExpression()) {
+        sourceTransformExpressions.addAll(analysis.getOrderByExpressions());
+      }
     }
+  }
 
-    sourceTransformExpressions.addAll(analysis.getSelectExpressions());
-    if (queryStatement.hasOrderByExpression()) {
-      sourceTransformExpressions.addAll(analysis.getOrderByExpressions());
+  private void putAggExpressionToSourceTransform(
+      Collection<Expression> aggExpressions, Analysis analysis) {
+    Set<Expression> sourceTransformExpressions = analysis.getSourceTransformExpressions();
+    for (Expression aggExpression : aggExpressions) {
+      // put countTimeSourceExpression of count_time aggregation into sourceTransformExpression
+      if (aggExpression instanceof FunctionExpression
+          && COUNT_TIME.equalsIgnoreCase(((FunctionExpression) aggExpression).getFunctionName())) {
+        for (Expression countTimeSourceExpression :
+            ((FunctionExpression) aggExpression).getCountTimeExpressions()) {
+          analyzeExpressionType(analysis, countTimeSourceExpression);
+          sourceTransformExpressions.add(countTimeSourceExpression);
+        }
+      } else {
+        // for AggregationExpression, only the first Expression of input need to transform
+        sourceTransformExpressions.add(aggExpression.getExpressions().get(0));
+      }
     }
   }
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/CountTimeAggregationAmountVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/CountTimeAggregationAmountVisitor.java
new file mode 100644
index 00000000000..c7de2d888f1
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/CountTimeAggregationAmountVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.expression.visitor;
+
+import org.apache.iotdb.db.queryengine.plan.expression.Expression;
+import org.apache.iotdb.db.queryengine.plan.expression.leaf.LeafOperand;
+import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME;
+
+public class CountTimeAggregationAmountVisitor extends CollectVisitor {
+
+  @Override
+  public List<Expression> visitFunctionExpression(
+      FunctionExpression functionExpression, Void context) {
+    if (COUNT_TIME.equalsIgnoreCase(functionExpression.getFunctionName())) {
+      return Collections.singletonList(functionExpression);
+    }
+    return Collections.emptyList();
+  }
+
+  @Override
+  public List<Expression> visitLeafOperand(LeafOperand leafOperand, Void context) {
+    return Collections.emptyList();
+  }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
index 6339bcb30e7..32b227cd1c4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer;
 import org.apache.iotdb.db.queryengine.plan.expression.Expression;
 import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
 import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.queryengine.plan.expression.visitor.CountTimeAggregationAmountVisitor;
 import org.apache.iotdb.db.queryengine.plan.statement.Statement;
 import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
 import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
@@ -51,6 +52,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME;
+
 /**
  * Base class of SELECT statement.
  *
@@ -486,38 +489,23 @@ public class QueryStatement extends Statement {
       if (isGroupByTag() && isAlignByDevice()) {
         throw new SemanticException("GROUP BY TAGS does not support align by device now.");
       }
-      boolean hasCountTimeAggregation = false;
       Set<String> outputColumn = new HashSet<>();
       for (ResultColumn resultColumn : selectComponent.getResultColumns()) {
         if (resultColumn.getColumnType() != ResultColumn.ColumnType.AGGREGATION) {
           throw new SemanticException(RAW_AGGREGATION_HYBRID_QUERY_ERROR_MSG);
         }
-        if (resultColumn.getExpression() instanceof FunctionExpression
-            && "count_time"
-                .equalsIgnoreCase(
-                    ((FunctionExpression) resultColumn.getExpression()).getFunctionName())) {
-          hasCountTimeAggregation = true;
-        }
         outputColumn.add(
             resultColumn.getAlias() != null
                 ? resultColumn.getAlias()
                 : resultColumn.getExpression().getExpressionString());
       }
 
-      if (hasCountTimeAggregation) {
-        if (isGroupByLevel()) {
-          throw new SemanticException(COUNT_TIME_NOT_SUPPORT_GROUP_BY_LEVEL);
-        }
-        if (isGroupByTag()) {
-          throw new SemanticException(COUNT_TIME_NOT_SUPPORT_GROUP_BY_TAG);
-        }
-      }
-
       for (Expression expression : getExpressionSortItemList()) {
         if (!hasAggregationFunction(expression)) {
           throw new SemanticException(RAW_AGGREGATION_HYBRID_QUERY_ERROR_MSG);
         }
       }
+      CountTimeAggregationAmountVisitor countTimeAggregationAmountVisitor = new CountTimeAggregationAmountVisitor();
       if (isGroupByTag()) {
         if (hasHaving()) {
           throw new SemanticException("Having clause is not supported yet in GROUP BY TAGS query");
@@ -532,6 +520,9 @@ public class QueryStatement extends Statement {
         }
         for (ResultColumn resultColumn : selectComponent.getResultColumns()) {
           Expression expression = resultColumn.getExpression();
+          if (countTimeAggregationAmountVisitor.process(resultColumn.getExpression(), null).size() > 0) {
+            throw new SemanticException(COUNT_TIME_NOT_SUPPORT_GROUP_BY_TAG);
+          }
           if (!(expression instanceof FunctionExpression
               && expression.getExpressions().get(0) instanceof TimeSeriesOperand
               && expression.isBuiltInAggregationFunctionExpression())) {
@@ -540,6 +531,17 @@ public class QueryStatement extends Statement {
           }
         }
       }
+      if (isGroupByLevel()
+          && selectComponent.getResultColumns().stream()
+              .anyMatch(
+                  resultColumn ->
+                      resultColumn
+                          .getExpression()
+                          .getOutputSymbol()
+                          .toLowerCase()
+                          .contains(COUNT_TIME))) {
+        throw new SemanticException(COUNT_TIME_NOT_SUPPORT_GROUP_BY_LEVEL);
+      }
       if (hasGroupByExpression()) {
         // Aggregation expression shouldn't exist in group by clause.
         List<Expression> aggregationExpression =