You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/09/18 04:32:41 UTC

[iotdb] branch load_v2 updated: add test

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/load_v2 by this push:
     new 0b190fb41f4 add test
0b190fb41f4 is described below

commit 0b190fb41f46be9f24886a9d36df41d5fbb88ab8
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Sep 18 12:36:42 2023 +0800

    add test
---
 .../execution/load/TsFileSplitSender.java          |  7 +-
 .../locseq/ThroughputBasedLocationSequencer.java   |  4 +-
 .../org/apache/iotdb/db/utils/SequenceUtils.java   | 78 ++++++++++++++++++++++
 3 files changed, 85 insertions(+), 4 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
index 41fdfd4bc66..e1382c254ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
@@ -46,6 +46,8 @@ import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
 import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationSequencer;
 import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationStatistics;
 import org.apache.iotdb.db.queryengine.execution.load.locseq.ThroughputBasedLocationSequencer;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.ClusteringMeasurementSplitter;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.OrderedMeasurementSplitter;
 import org.apache.iotdb.db.queryengine.execution.load.nodesplit.PieceNodeSplitter;
 import org.apache.iotdb.db.queryengine.execution.load.nodesplit.PieceNodeSplitter.SingletonSplitter;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode;
@@ -81,7 +83,8 @@ public class TsFileSplitSender {
       new ConcurrentHashMap<>();
   private Map<TRegionReplicaSet, Exception> phaseTwoFailures = new HashMap<>();
   private long maxSplitSize;
-  private PieceNodeSplitter pieceNodeSplitter = new SingletonSplitter();
+  private PieceNodeSplitter pieceNodeSplitter = new ClusteringMeasurementSplitter(5, 10);
+//  private PieceNodeSplitter pieceNodeSplitter = new OrderedMeasurementSplitter();
 
   public TsFileSplitSender(
       LoadTsFileNode loadTsFileNode,
@@ -217,7 +220,7 @@ public class TsFileSplitSender {
       Exception lastConnectionError = null;
       TDataNodeLocation currLocation = null;
       for (TDataNodeLocation location : locationSequencer) {
-        if (location.getDataNodeId() == 0) {
+        if (location.getDataNodeId() == 0 && logger.isDebugEnabled()) {
           locationStatistics.logLocationStatistics();
           logger.info("Chose location {}", location.getDataNodeId());
         }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
index 66929c2d350..48cf1fa2c1b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
@@ -114,8 +114,8 @@ public class ThroughputBasedLocationSequencer implements LocationSequencer {
         chosen = i;
       }
     }
-    if (chosen == 0 && locations.size() == 3) {
-      logger.info("Dice {}, chosen {}, ranks {}", dice, chosen,
+    if (chosen == 0 && locations.size() == 3 && logger.isDebugEnabled()) {
+      logger.debug("Dice {}, chosen {}, ranks {}", dice, chosen,
           locations.stream().map(p -> new Pair<>(p.left.getDataNodeId(), p.right)).collect(
               Collectors.toList()));
     }
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java
new file mode 100644
index 00000000000..494737f0446
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import java.util.Random;
+
+public class SequenceUtils {
+  public interface DoubleSequenceGenerator {
+    double gen(int id);
+  }
+
+  public interface DoubleSequenceGeneratorFactory {
+    DoubleSequenceGenerator create();
+  }
+
+  public static class SimpleDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+    @Override
+    public double gen(int id) {
+      return id * 1.0;
+    }
+
+    public static class Factory implements DoubleSequenceGeneratorFactory{
+      @Override
+      public DoubleSequenceGenerator create() {
+        return new SimpleDoubleSequenceGenerator();
+      }
+    }
+  }
+
+  public static class UniformDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+    private Random random = new Random();
+    @Override
+    public double gen(int id) {
+      return random.nextDouble();
+    }
+
+    public static class Factory implements DoubleSequenceGeneratorFactory{
+      @Override
+      public DoubleSequenceGenerator create() {
+        return new UniformDoubleSequenceGenerator();
+      }
+    }
+  }
+
+  public static class GaussianDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+    private Random random = new Random();
+    @Override
+    public double gen(int id) {
+      return random.nextGaussian();
+    }
+
+    public static class Factory implements DoubleSequenceGeneratorFactory{
+      @Override
+      public DoubleSequenceGenerator create() {
+        return new GaussianDoubleSequenceGenerator();
+      }
+    }
+  }
+}