You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/09/18 04:32:41 UTC
[iotdb] branch load_v2 updated: add test
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/load_v2 by this push:
new 0b190fb41f4 add test
0b190fb41f4 is described below
commit 0b190fb41f46be9f24886a9d36df41d5fbb88ab8
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Sep 18 12:36:42 2023 +0800
add test
---
.../execution/load/TsFileSplitSender.java | 7 +-
.../locseq/ThroughputBasedLocationSequencer.java | 4 +-
.../org/apache/iotdb/db/utils/SequenceUtils.java | 78 ++++++++++++++++++++++
3 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
index 41fdfd4bc66..e1382c254ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
@@ -46,6 +46,8 @@ import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationSequencer;
import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationStatistics;
import org.apache.iotdb.db.queryengine.execution.load.locseq.ThroughputBasedLocationSequencer;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.ClusteringMeasurementSplitter;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.OrderedMeasurementSplitter;
import org.apache.iotdb.db.queryengine.execution.load.nodesplit.PieceNodeSplitter;
import org.apache.iotdb.db.queryengine.execution.load.nodesplit.PieceNodeSplitter.SingletonSplitter;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode;
@@ -81,7 +83,8 @@ public class TsFileSplitSender {
new ConcurrentHashMap<>();
private Map<TRegionReplicaSet, Exception> phaseTwoFailures = new HashMap<>();
private long maxSplitSize;
- private PieceNodeSplitter pieceNodeSplitter = new SingletonSplitter();
+ private PieceNodeSplitter pieceNodeSplitter = new ClusteringMeasurementSplitter(5, 10);
+// private PieceNodeSplitter pieceNodeSplitter = new OrderedMeasurementSplitter();
public TsFileSplitSender(
LoadTsFileNode loadTsFileNode,
@@ -217,7 +220,7 @@ public class TsFileSplitSender {
Exception lastConnectionError = null;
TDataNodeLocation currLocation = null;
for (TDataNodeLocation location : locationSequencer) {
- if (location.getDataNodeId() == 0) {
+ if (location.getDataNodeId() == 0 && logger.isDebugEnabled()) {
locationStatistics.logLocationStatistics();
logger.info("Chose location {}", location.getDataNodeId());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
index 66929c2d350..48cf1fa2c1b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/locseq/ThroughputBasedLocationSequencer.java
@@ -114,8 +114,8 @@ public class ThroughputBasedLocationSequencer implements LocationSequencer {
chosen = i;
}
}
- if (chosen == 0 && locations.size() == 3) {
- logger.info("Dice {}, chosen {}, ranks {}", dice, chosen,
+ if (chosen == 0 && locations.size() == 3 && logger.isDebugEnabled()) {
+ logger.debug("Dice {}, chosen {}, ranks {}", dice, chosen,
locations.stream().map(p -> new Pair<>(p.left.getDataNodeId(), p.right)).collect(
Collectors.toList()));
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java
new file mode 100644
index 00000000000..494737f0446
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SequenceUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import java.util.Random;
+
+public class SequenceUtils {
+ public interface DoubleSequenceGenerator {
+ double gen(int id);
+ }
+
+ public interface DoubleSequenceGeneratorFactory {
+ DoubleSequenceGenerator create();
+ }
+
+ public static class SimpleDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+ @Override
+ public double gen(int id) {
+ return id * 1.0;
+ }
+
+ public static class Factory implements DoubleSequenceGeneratorFactory{
+ @Override
+ public DoubleSequenceGenerator create() {
+ return new SimpleDoubleSequenceGenerator();
+ }
+ }
+ }
+
+ public static class UniformDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+ private Random random = new Random();
+ @Override
+ public double gen(int id) {
+ return random.nextDouble();
+ }
+
+ public static class Factory implements DoubleSequenceGeneratorFactory{
+ @Override
+ public DoubleSequenceGenerator create() {
+ return new UniformDoubleSequenceGenerator();
+ }
+ }
+ }
+
+ public static class GaussianDoubleSequenceGenerator implements DoubleSequenceGenerator {
+
+ private Random random = new Random();
+ @Override
+ public double gen(int id) {
+ return random.nextGaussian();
+ }
+
+ public static class Factory implements DoubleSequenceGeneratorFactory{
+ @Override
+ public DoubleSequenceGenerator create() {
+ return new GaussianDoubleSequenceGenerator();
+ }
+ }
+ }
+}