You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by do...@apache.org on 2019/04/19 09:29:08 UTC

[incubator-iotdb] branch cluster updated: add config file generator

This is an automated email from the ASF dual-hosted git repository.

dope pushed a commit to branch cluster
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git


The following commit(s) were added to refs/heads/cluster by this push:
     new 6df0565  add config file generator
     new 5a1d73e  Merge branch 'cluster' of github.com:apache/incubator-iotdb into cluster
6df0565 is described below

commit 6df05651d336957c3d34d38d17c4025d03d65b29
Author: MyXOF <xu...@163.com>
AuthorDate: Fri Apr 19 17:28:26 2019 +0800

    add config file generator
---
 cluster/script/deploy.sh                           | 57 +++++----------------
 cluster/script/stop.sh                             | 21 ++++++--
 .../integration/IoTDBMetadataFetchRemoteIT.java    | 58 +++-------------------
 .../cluster/utils/ClusterConfigureGenerator.java   | 27 ++++------
 4 files changed, 44 insertions(+), 119 deletions(-)

diff --git a/cluster/script/deploy.sh b/cluster/script/deploy.sh
index bbd1028..9873632 100755
--- a/cluster/script/deploy.sh
+++ b/cluster/script/deploy.sh
@@ -19,43 +19,23 @@
 #
 
 
-## ========== package file locally ==========
-## cd ../../
-#cd iotdb/iotdb
-#rm -rf data/
-#rm -rf lib/
-#rm -rf lib_cluster/
-#rm -rf logs/
-#cd ../..
-#git pull
-#mvn clean package -pl cluster -am -Dmaven.test.skip=true
-#cd iotdb
-#rm cluster.zip
-#zip -r cluster.zip iotdb/
-#cd ..
-
-## ========== package file locally end ==========
-
-
-## ========== transfer to node ==========
-
-#alliplist=('192.168.130.12' '192.168.130.14' '192.168.130.16' '192.168.130.18' '192.168.130.19')
-#for ip in ${alliplist[@]}
-#do
-#  ssh fit@$ip "rm /home/fit/xuyi/cluster.zip"
-#  ssh fit@$ip "rm -rf /home/fit/xuyi/iotdb"
-#  scp iotdb/cluster.zip fit@$ip:/home/fit/xuyi
-#  ssh fit@$ip "unzip -o -d /home/fit/xuyi /home/fit/xuyi/cluster.zip "
-#done
-## ========== transfer to node end ==========
-
-
+## Only for test
 nodes=$1
 if [ $nodes == "3" ]
 then
   iplist=('192.168.130.14' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "5" ]
+then
+  iplist=('192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "7" ]
+then
+  iplist=('192.168.130.9' '192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.15' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "10" ]
+then
+  iplist=('192.168.130.8' '192.168.130.9' '192.168.130.10' '192.168.130.9' '192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.15' '192.168.130.16' '192.168.130.18')
 else
-  iplist=('192.168.130.12' '192.168.130.14' '192.168.130.16' '192.168.130.18' '192.168.130.19')
+  echo "node number error"
+  exit 1;
 fi
 
 replication=$2
@@ -65,16 +45,5 @@ for ip in ${iplist[@]}
 do
   idx="$(cut -d'.' -f4 <<<"$ip")"
   cat $3/src/test/resources/conf/$nodes-$replication-$idx.properties
-#  scp $3/src/test/resources/conf/$nodes-$replication-$idx.properties fit@$ip:/home/fit/xuyi/iotdb/conf/iotdb-cluster.properties
+  scp $3/src/test/resources/conf/$nodes-$replication-$idx.properties fit@$ip:/home/fit/xuyi/incubator-iotdb/iotdb/iotdb/conf/iotdb-cluster.properties
 done
-
-for ip in ${iplist[@]}
-do
-  ssh fit@$ip "chmod a+x /home/fit/xuyi/iotdb/bin/stop-cluster.sh"
-  ssh fit@$ip "sh /home/fit/xuyi/iotdb/bin/stop-cluster.sh"
-  ssh fit@$ip "rm -rf /home/fit/xuyi/iotdb/data"
-  ssh fit@$ip "rm -rf /home/fit/xuyi/iotdb/logs"
-  ssh fit@$ip "chmod a+x /home/fit/xuyi/iotdb/bin/start-cluster.sh"
-  ssh fit@$ip "nohup /home/fit/xuyi/iotdb/bin/start-cluster.sh > output.log 2>&1 &"
-done
-
diff --git a/cluster/script/stop.sh b/cluster/script/stop.sh
index b4cdf41..c54784b 100755
--- a/cluster/script/stop.sh
+++ b/cluster/script/stop.sh
@@ -18,19 +18,30 @@
 # under the License.
 #
 
+## Only for test
 nodes=$1
 if [ $nodes == "3" ]
 then
   iplist=('192.168.130.14' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "5" ]
+then
+  iplist=('192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "7" ]
+then
+  iplist=('192.168.130.9' '192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.15' '192.168.130.16' '192.168.130.18')
+elif [ $nodes == "10" ]
+then
+  iplist=('192.168.130.8' '192.168.130.9' '192.168.130.10' '192.168.130.9' '192.168.130.12' '192.168.130.13' '192.168.130.14' '192.168.130.15' '192.168.130.16' '192.168.130.18')
 else
-  iplist=('192.168.130.12' '192.168.130.14' '192.168.130.16' '192.168.130.18' '192.168.130.19')
+  echo "node number error"
+  exit 1;
 fi
 
 for ip in ${iplist[@]}
 do
-  ssh fit@$ip "chmod a+x /home/fit/xuyi/iotdb/bin/stop-cluster.sh"
-  ssh fit@$ip "sh /home/fit/xuyi/iotdb/bin/stop-cluster.sh"
-  ssh fit@$ip "rm -rf /home/fit/xuyi/iotdb/data"
-  ssh fit@$ip "rm -rf /home/fit/xuyi/iotdb/logs"
+  ssh fit@$ip "chmod a+x /home/fit/xuyi/incubator-iotdb/iotdb/iotdb/bin/stop-cluster.sh"
+  ssh fit@$ip "sh /home/fit/xuyi/incubator-iotdb/iotdb/iotdb/bin/stop-cluster.sh"
+  ssh fit@$ip "rm -rf /home/fit/xuyi/incubator-iotdb/iotdb/iotdb/data"
+  ssh fit@$ip "rm -rf /home/fit/xuyi/incubator-iotdb/iotdb/iotdb/logs"
 done
 
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/integration/IoTDBMetadataFetchRemoteIT.java b/cluster/src/test/java/org/apache/iotdb/cluster/integration/IoTDBMetadataFetchRemoteIT.java
index 681402b..ccb1456 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/integration/IoTDBMetadataFetchRemoteIT.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/integration/IoTDBMetadataFetchRemoteIT.java
@@ -19,10 +19,11 @@
 package org.apache.iotdb.cluster.integration;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.sql.SQLException;
 import org.apache.iotdb.cluster.utils.ClusterConfigureGenerator;
+import org.apache.iotdb.cluster.utils.Utils;
 import org.apache.iotdb.jdbc.Config;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -41,57 +42,10 @@ public class IoTDBMetadataFetchRemoteIT extends IoTDBMetadataFetchAbstract{
   }
 
   @Test
-  public void test3Node3Replicator() throws SQLException, IOException {
-//    final String URL = "192.168.130.16:6667/";
-//    String dir = Utils.getCurrentPath("pwd");
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "3", "3", dir);
-//    test(URL, false);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "3", dir);
-
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "3", "3", dir);
-//    test(URL, true);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "3", dir);
-  }
-
-
-  @Test
-  public void test3Node1Replicator() throws SQLException, IOException {
-//    final String URL = "192.168.130.16:6667/";
-//    String dir = Utils.getCurrentPath("pwd");
-//
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "3", "1", dir);
-//    test(URL, false);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "3", dir);
-//
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "3", "1", dir);
-//    test(URL, true);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "3", dir);
-  }
-
-  @Test
-  public void test5Node3Replicator() throws SQLException, IOException {
-//    final String URL = "192.168.130.16:6667/";
-//    String dir = Utils.getCurrentPath("pwd");
-//
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "5", "3", dir);
-//    test(URL, false);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "5", dir);
-//
-//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh",
-//        "5", "3", dir);
-//    test(URL, true);
-//    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh",
-//        "5", dir);
+  public void test() throws IOException {
+    String dir = Utils.getCurrentPath("pwd");
+//    startScript("sh", dir + File.separator + "script" + File.separator + "deploy.sh", "7", "1", dir);
+    startScript("sh", dir + File.separator + "script" + File.separator + "stop.sh", "7", "3", dir);
   }
 
   private void startScript(String... command) throws IOException{
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/utils/ClusterConfigureGenerator.java b/cluster/src/test/java/org/apache/iotdb/cluster/utils/ClusterConfigureGenerator.java
index 85eb30d..d8ed728 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/utils/ClusterConfigureGenerator.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/utils/ClusterConfigureGenerator.java
@@ -32,23 +32,9 @@ public class ClusterConfigureGenerator {
 
   private static final String TEST_CONF_PATH = "src" + File.separatorChar + "test"
       + File.separatorChar + "resources" + File.separatorChar + "conf";
-  private static ClusterConfig config = ClusterDescriptor.getInstance().getConfig();
-  private static Map<String, String> configMap = new HashMap<String, String>() {
-    {
-      put("port", config.getPort() + "");
-      put("election_timeout_ms", config.getElectionTimeoutMs() + "");
-      put("max_catch_up_log_num", config.getMaxCatchUpLogNum() + "");
-      put("delay_snapshot", config.isDelaySnapshot() + "");
-      put("delay_hours", config.getDelayHours() + "");
-      put("task_redo_count", config.getQpTaskRedoCount() + "");
-      put("task_timeout_ms", config.getQpTaskTimeout() + "");
-      put("num_of_virtual_nodes", config.getNumOfVirtualNodes() + "");
-      put("max_num_of_inner_rpc_client", config.getMaxNumOfInnerRpcClient() + "");
-      put("max_queue_num_of_inner_rpc_client", config.getMaxQueueNumOfInnerRpcClient() + "");
-      put("read_metadata_consistency_level", config.getReadMetadataConsistencyLevel() + "");
-      put("read_data_consistency_level", config.getReadDataConsistencyLevel() + "");
-    }
-  };
+  private static Map<String, String> configMap = new HashMap<String, String>() {{
+    put("qp_task_timeout_ms", "10000");
+  }};
 
   public static void generateClusterConfigure() throws IOException {
     deleteClusterConfigure();
@@ -57,7 +43,12 @@ public class ClusterConfigureGenerator {
     String[][] config = {
         {"192.168.130.14:8888,192.168.130.16:8888,192.168.130.18:8888", "1"},
         {"192.168.130.14:8888,192.168.130.16:8888,192.168.130.18:8888", "3"},
-        {"192.168.130.12:8888,192.168.130.14:8888,192.168.130.16:8888,192.168.130.18:8888,192.168.130.19:8888", "3"},
+        {"192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.16:8888,192.168.130.18:8888", "1"},
+        {"192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.16:8888,192.168.130.18:8888", "3"},
+        {"192.168.130.9:8888,192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.15:8888,192.168.130.16:8888,192.168.130.18:8888", "1"},
+        {"192.168.130.9:8888,192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.15:8888,192.168.130.16:8888,192.168.130.18:8888", "3"},
+        {"192.168.130.8:8888,192.168.130.9:8888,192.168.130.10:8888,192.168.130.7:8888,192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.15:8888,192.168.130.16:8888,192.168.130.18:8888", "3"},
+        {"192.168.130.8:8888,192.168.130.9:8888,192.168.130.10:8888,192.168.130.7:8888,192.168.130.12:8888,192.168.130.13:8888,192.168.130.14:8888,192.168.130.15:8888,192.168.130.16:8888,192.168.130.18:8888", "5"},
     };
     for(String[] c : config){
       for(String node : c[0].split(",")){