You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2020/01/10 09:26:26 UTC

[incubator-iotdb] branch master updated: [IOTDB-412]Paths are not correctly deduplicated (#727)

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 279ff01  [IOTDB-412]Paths are not correctly deduplicated (#727)
279ff01 is described below

commit 279ff019a8a94a93f58d19c82150bb943e9952ff
Author: Dawei Liu <at...@163.com>
AuthorDate: Fri Jan 10 17:26:16 2020 +0800

    [IOTDB-412]Paths are not correctly deduplicated (#727)
    
    * add new ut
    
    * add comment
---
 .../iotdb/db/qp/strategy/PhysicalGenerator.java    | 34 +++++++------
 .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java  | 59 ++++++++++++++++++++--
 2 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index 56660cd..47b09af 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -139,7 +139,8 @@ public class PhysicalGenerator {
           case SQLConstant.TOK_STORAGE_GROUP:
             return new ShowPlan(ShowContentType.STORAGE_GROUP);
           case SQLConstant.TOK_DEVICES:
-            return new ShowDevicesPlan(ShowContentType.DEVICES, ((ShowDevicesOperator) operator).getPath());
+            return new ShowDevicesPlan(ShowContentType.DEVICES,
+                ((ShowDevicesOperator) operator).getPath());
           case SQLConstant.TOK_COUNT_NODE_TIMESERIES:
             return new CountPlan(ShowContentType.COUNT_NODE_TIMESERIES,
                 ((CountOperator) operator).getPath(), ((CountOperator) operator).getLevel());
@@ -225,7 +226,8 @@ public class PhysicalGenerator {
           Path fullPath = Path.addPrefixPath(suffixPath, prefixPath);
           Set<String> tmpDeviceSet = new HashSet<>();
           try {
-            List<String> actualPaths = executor.getAllMatchedPaths(fullPath.getFullPath());  // remove stars to get actual paths
+            List<String> actualPaths = executor
+                .getAllMatchedPaths(fullPath.getFullPath());  // remove stars to get actual paths
             for (String pathStr : actualPaths) {
               Path path = new Path(pathStr);
               String device = path.getDevice();
@@ -259,9 +261,9 @@ public class PhysicalGenerator {
               if (dataTypeConsistencyChecker.containsKey(measurementChecked)) {
                 if (!dataType.equals(dataTypeConsistencyChecker.get(measurementChecked))) {
                   throw new QueryProcessException(
-                          "The data types of the same measurement column should be the same across "
-                                  + "devices in GROUP_BY_DEVICE sql. For more details please refer to the "
-                                  + "SQL document.");
+                      "The data types of the same measurement column should be the same across "
+                          + "devices in GROUP_BY_DEVICE sql. For more details please refer to the "
+                          + "SQL document.");
                 }
               } else {
                 dataTypeConsistencyChecker.put(measurementChecked, dataType);
@@ -313,12 +315,11 @@ public class PhysicalGenerator {
       queryPlan.setMeasurementsGroupByDevice(measurementsGroupByDevice);
       queryPlan.setDataTypeConsistencyChecker(dataTypeConsistencyChecker);
       queryPlan.setPaths(paths);
-      queryPlan.setDeduplicatedPaths(paths);
 
       // get device to filter map
       FilterOperator filterOperator = queryOperator.getFilterOperator();
 
-      if(filterOperator != null){
+      if (filterOperator != null) {
         queryPlan.setDeviceToFilterMap(concatFilterByDivice(prefixPaths, filterOperator));
       }
     } else {
@@ -345,8 +346,9 @@ public class PhysicalGenerator {
   // e.g. translate "select * from root.ln.d1, root.ln.d2 where s1 < 20 AND s2 > 10" to
   // [root.ln.d1 -> root.ln.d1.s1 < 20 AND root.ln.d1.s2 > 10,
   //  root.ln.d2 -> root.ln.d2.s1 < 20 AND root.ln.d2.s2 > 10)]
-  private Map<String, IExpression> concatFilterByDivice(List<Path> fromPaths, FilterOperator operator)
-          throws QueryProcessException {
+  private Map<String, IExpression> concatFilterByDivice(List<Path> fromPaths,
+      FilterOperator operator)
+      throws QueryProcessException {
     Map<String, IExpression> deviceToFilterMap = new HashMap<>();
     // remove stars in fromPaths and get deviceId with deduplication
     List<String> noStarDevices = removeStarsInDeviceWithUnique(fromPaths);
@@ -361,7 +363,7 @@ public class PhysicalGenerator {
   }
 
   private List<String> removeStarsInDeviceWithUnique(List<Path> paths)
-          throws LogicalOptimizeException {
+      throws LogicalOptimizeException {
     List<String> retDevices;
     Set<String> deviceSet = new LinkedHashSet<>();
     try {
@@ -383,7 +385,7 @@ public class PhysicalGenerator {
   }
 
   private FilterOperator concatFilterPath(String prefix, FilterOperator operator) {
-    if(!operator.isLeaf()){
+    if (!operator.isLeaf()) {
       for (FilterOperator child : operator.getChildren()) {
         concatFilterPath(prefix, child);
       }
@@ -416,11 +418,13 @@ public class PhysicalGenerator {
   }
 
   private void deduplicate(QueryPlan queryPlan) {
+    //The deduplication of a GroupByDevice query is done in the dataset
+    if (queryPlan.isGroupByDevice()) {
+      return;
+    }
     if (queryPlan instanceof AggregationPlan) {
-      if (!queryPlan.isGroupByDevice()) {
-        AggregationPlan aggregationPlan = (AggregationPlan) queryPlan;
-        deduplicateAggregation(aggregationPlan);
-      }
+      AggregationPlan aggregationPlan = (AggregationPlan) queryPlan;
+      deduplicateAggregation(aggregationPlan);
       return;
     }
     List<Path> paths = queryPlan.getPaths();
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java b/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
index c55e1c1..9f38f50 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
@@ -23,8 +23,10 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
+import org.apache.iotdb.db.exception.StartupException;
 import org.apache.iotdb.db.exception.metadata.MetadataException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.MManager;
 import org.apache.iotdb.db.qp.QueryProcessor;
 import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
 import org.apache.iotdb.db.qp.physical.PhysicalPlan;
@@ -54,6 +56,7 @@ import org.apache.iotdb.tsfile.read.filter.TimeFilter;
 import org.apache.iotdb.tsfile.read.filter.ValueFilter;
 import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
 import org.apache.iotdb.tsfile.utils.StringContainer;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -63,7 +66,7 @@ public class PhysicalPlanTest {
   private QueryProcessor processor = new QueryProcessor(new MemIntQpExecutor());
 
   @Before
-  public void before() throws QueryProcessException {
+  public void before() throws QueryProcessException, StartupException {
     Path path1 = new Path(
         new StringContainer(new String[]{"root", "vehicle", "d1", "s1"},
             TsFileConstant.PATH_SEPARATOR));
@@ -84,6 +87,12 @@ public class PhysicalPlanTest {
         .insert(new InsertPlan(path3.getDevice(), 10, path3.getMeasurement(), "10"));
     processor.getExecutor()
         .insert(new InsertPlan(path4.getDevice(), 10, path4.getMeasurement(), "10"));
+    MManager.getInstance().init();
+  }
+
+  @After
+  public void clean() {
+    MManager.getInstance().clear();
   }
 
   @Test
@@ -178,10 +187,10 @@ public class PhysicalPlanTest {
 
   @Test
   public void testGroupBy3()
-          throws QueryProcessException, MetadataException {
+      throws QueryProcessException, MetadataException {
     String sqlStr =
-            "select count(s1) " + "from root.vehicle.d1 " + "where s1 < 20 and time <= now() "
-                    + "group by([2017-6-2T12:00:12+07:00,2017-6-12T12:00:12+07:00], 3h, 24h)";
+        "select count(s1) " + "from root.vehicle.d1 " + "where s1 < 20 and time <= now() "
+            + "group by([2017-6-2T12:00:12+07:00,2017-6-12T12:00:12+07:00], 3h, 24h)";
     PhysicalPlan plan = processor.parseSQLToPhysicalPlan(sqlStr);
     if (!plan.isQuery()) {
       fail();
@@ -595,7 +604,47 @@ public class PhysicalPlanTest {
     QueryProcessor processor = new QueryProcessor(new MemIntQpExecutor());
     OperateFilePlan plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
     assertEquals(
-        String.format("OperateFilePlan{file=%s, targetDir=%s, autoCreateSchema=false, sgLevel=0, operatorType=MOVE_FILE}", filePath,
+        String.format(
+            "OperateFilePlan{file=%s, targetDir=%s, autoCreateSchema=false, sgLevel=0, operatorType=MOVE_FILE}",
+            filePath,
             targetDir), plan.toString());
   }
+
+  @Test
+  public void testDeduplicatedPath() throws Exception {
+    String sqlStr = "select * from root.sg.d1,root.sg.d1,root.sg.d1";
+    QueryPlan plan = (QueryPlan) processor.parseSQLToPhysicalPlan(sqlStr);
+    Assert.assertEquals(1, plan.getDeduplicatedPaths().size());
+    Assert.assertEquals(1, plan.getDeduplicatedDataTypes().size());
+    Assert.assertEquals(new Path("root.sg.d1.*"), plan.getDeduplicatedPaths().get(0));
+
+    sqlStr = "select count(*) from root.sg.d1,root.sg.d1,root.sg.d1";
+    plan = (QueryPlan) processor.parseSQLToPhysicalPlan(sqlStr);
+    Assert.assertEquals(1, plan.getDeduplicatedPaths().size());
+    Assert.assertEquals(1, plan.getDeduplicatedDataTypes().size());
+    Assert.assertEquals(new Path("root.sg.d1.*"), plan.getDeduplicatedPaths().get(0));
+
+    //'group by device' is deduplication in DeviceIterateDataSet
+    MManager manager = MManager.getInstance();
+    manager.setStorageGroupToMTree("root.vehicle");
+    manager.addPathToMTree("root.vehicle.d0.s1", "INT64", "PLAIN");
+    manager.addPathToMTree("root.vehicle.d0.s0", "INT64", "PLAIN");
+    manager.addPathToMTree("root.vehicle.d1.s0", "INT64", "PLAIN");
+    manager.addPathToMTree("root.vehicle.d1.s1", "INT64", "PLAIN");
+
+    sqlStr = "select s0,s0,s1 from root.vehicle.d0, root.vehicle.d1 group by device";
+    plan = (QueryPlan) processor.parseSQLToPhysicalPlan(sqlStr);
+    Assert.assertEquals(0, plan.getDeduplicatedPaths().size());
+    Assert.assertEquals(0, plan.getDeduplicatedDataTypes().size());
+    Assert.assertEquals(6, plan.getPaths().size());
+    Assert.assertEquals(6, plan.getDataTypes().size());
+
+    sqlStr = "select COUNT(s0),COUNT(s0),COUNT(s1) from root.vehicle.d0, root.vehicle.d1 group by device";
+    plan = (QueryPlan) processor.parseSQLToPhysicalPlan(sqlStr);
+    Assert.assertEquals(0, plan.getDeduplicatedPaths().size());
+    Assert.assertEquals(0, plan.getDeduplicatedPaths().size());
+    Assert.assertEquals(0, plan.getDeduplicatedDataTypes().size());
+    Assert.assertEquals(6, plan.getPaths().size());
+    Assert.assertEquals(6, plan.getDataTypes().size());
+  }
 }