You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by er...@apache.org on 2022/11/01 05:29:16 UTC
[iotdb] branch improve/iotdb-4814 created (now cc6c23ca18)
This is an automated email from the ASF dual-hosted git repository.
ericpai pushed a change to branch improve/iotdb-4814
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at cc6c23ca18 [IOTDB-4814] Using 1C1D as the default mode in integration test
This branch includes the following new commits:
new cc6c23ca18 [IOTDB-4814] Using 1C1D as the default mode in integration test
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-4814] Using 1C1D as the default mode in integration test
Posted by er...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ericpai pushed a commit to branch improve/iotdb-4814
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cc6c23ca18df1f43e570d4ea7e68d91500e57503
Author: ericpai <er...@hotmail.com>
AuthorDate: Tue Nov 1 13:27:09 2022 +0800
[IOTDB-4814] Using 1C1D as the default mode in integration test
---
.../{cluster-it.yml => cluster-it-1c1d.yml} | 12 ++--
.../{cluster-it.yml => cluster-it-1c3d.yml} | 2 +-
.github/workflows/standalone-it-for-mpp.yml | 79 ----------------------
integration-test/README.md | 50 +++++++-------
integration-test/pom.xml | 23 ++-----
.../org/apache/iotdb/it/env/ConfigFactory.java | 41 ++++-------
.../java/org/apache/iotdb/it/env/EnvFactory.java | 20 +++---
...StandaloneDataNodeWrapper.java => EnvType.java} | 16 ++---
.../{StandaloneOnMppEnv.java => OneCopyEnv.java} | 22 +-----
9 files changed, 72 insertions(+), 193 deletions(-)
diff --git a/.github/workflows/cluster-it.yml b/.github/workflows/cluster-it-1c1d.yml
similarity index 93%
copy from .github/workflows/cluster-it.yml
copy to .github/workflows/cluster-it-1c1d.yml
index 323bb31829..70de9c618c 100644
--- a/.github/workflows/cluster-it.yml
+++ b/.github/workflows/cluster-it-1c1d.yml
@@ -1,14 +1,12 @@
-name: New Cluster IT
+name: Cluster IT - OneCopy
on:
push:
branches:
- - master
paths-ignore:
- 'docs/**'
pull_request:
branches:
- - master
paths-ignore:
- 'docs/**'
# allow manually run the action:
@@ -22,7 +20,7 @@ env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
jobs:
- ClusterIT:
+ StandaloneMppIT:
strategy:
fail-fast: false
max-parallel: 20
@@ -68,12 +66,14 @@ jobs:
run: |
mvn clean verify \
-DskipUTs \
+ -DintegrationTest.forkCount=2 \
-pl integration-test \
- -am -PClusterIT
+ -am
- name: Upload Artifact
if: failure()
uses: actions/upload-artifact@v3
with:
- name: cluster-log-java${{ matrix.java }}-${{ runner.os }}
+ name: standalone-log-java${{ matrix.java }}-${{ runner.os }}
path: integration-test/target/cluster-logs
retention-days: 1
+
diff --git a/.github/workflows/cluster-it.yml b/.github/workflows/cluster-it-1c3d.yml
similarity index 99%
rename from .github/workflows/cluster-it.yml
rename to .github/workflows/cluster-it-1c3d.yml
index 323bb31829..5c5e9d3fa5 100644
--- a/.github/workflows/cluster-it.yml
+++ b/.github/workflows/cluster-it-1c3d.yml
@@ -1,4 +1,4 @@
-name: New Cluster IT
+name: Cluster IT - 1C3D
on:
push:
diff --git a/.github/workflows/standalone-it-for-mpp.yml b/.github/workflows/standalone-it-for-mpp.yml
deleted file mode 100644
index 7821890738..0000000000
--- a/.github/workflows/standalone-it-for-mpp.yml
+++ /dev/null
@@ -1,79 +0,0 @@
-#name: New Standalone IT
-#
-#on:
-## push:
-## branches:
-## paths-ignore:
-## - 'docs/**'
-## pull_request:
-## branches:
-## paths-ignore:
-## - 'docs/**'
-## # allow manually run the action:
-## workflow_dispatch:
-#
-#concurrency:
-# group: ${{ github.workflow }}-${{ github.ref }}
-# cancel-in-progress: true
-#
-#env:
-# MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-#
-#jobs:
-# StandaloneMppIT:
-# strategy:
-# fail-fast: false
-# max-parallel: 20
-# matrix:
-# java: [ 8, 11, 17 ]
-# os: [ ubuntu-latest, macos-latest, windows-latest ]
-# runs-on: ${{ matrix.os }}
-#
-# steps:
-# - uses: actions/checkout@v2
-# - name: Set up JDK ${{ matrix.java }}
-# uses: actions/setup-java@v1
-# with:
-# java-version: ${{ matrix.java }}
-# - name: Cache Maven packages
-# uses: actions/cache@v2
-# with:
-# path: ~/.m2
-# key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-# restore-keys: ${{ runner.os }}-m2-
-# - name: Check Apache Rat
-# run: mvn -B apache-rat:check -P site -P code-coverage
-# - name: Adjust network dynamic TCP ports range
-# if: ${{ runner.os == 'Windows' }}
-# shell: pwsh
-# run: |
-# netsh int ipv4 set dynamicport tcp start=32768 num=32768
-# netsh int ipv4 set dynamicport udp start=32768 num=32768
-# netsh int ipv6 set dynamicport tcp start=32768 num=32768
-# netsh int ipv6 set dynamicport udp start=32768 num=32768
-# - name: Adjust Linux kernel somaxconn
-# if: ${{ runner.os == 'Linux' }}
-# shell: bash
-# run: sudo sysctl -w net.core.somaxconn=65535
-# - name: Adjust Mac kernel somaxconn
-# if: ${{ runner.os == 'macOS' }}
-# shell: bash
-# run: sudo sysctl -w kern.ipc.somaxconn=65535
-# - name: IT/UT Test
-# shell: bash
-# # we do not compile client-cpp for saving time, it is tested in client.yml
-# # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
-# run: |
-# mvn clean verify \
-# -DskipUTs \
-# -DintegrationTest.forkCount=2 \
-# -pl integration-test \
-# -am -PLocalStandaloneOnMppIT
-# - name: Upload Artifact
-# if: failure()
-# uses: actions/upload-artifact@v3
-# with:
-# name: standalone-log-java${{ matrix.java }}-${{ runner.os }}
-# path: integration-test/target/cluster-logs
-# retention-days: 1
-#
diff --git a/integration-test/README.md b/integration-test/README.md
index f2c8526b2d..f0bec7cd0a 100644
--- a/integration-test/README.md
+++ b/integration-test/README.md
@@ -19,31 +19,32 @@
-->
-Integration Test For the MPP Architecture
-===================
+# Integration Test User Guide
-Integration test for the mpp architecture are in this module.
+Integration tests for IoTDB are in this module.
-Now integration testing supports the Cluster mode and the Local Standalone mode.
+Now integration testing supports two kinds of architecture.
-Integration Testing with Cluster Mode
--------------------
+- `OneCopy`: A cluster with 1 config node and 1 data node.
+- `Cluster1`: A cluster with 1 config node and 3 data nodes.
-You can run the integration test in cluster mode. At present, we have implemented a pseudo cluster with 1 config nodes and 3 data nodes.
-(As the test cases and the test environment are decoupled, we can easily implement other pseudo cluster or even a docker-based cluster later.)
+## Integration Testing with One Copy Mode
+
+Integration testing in `OneCopy` mode can be run with both maven and IDEs like IntelliJ easily.
The maven command is:
```
-mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT
+mvn clean verify -DskipUTs -pl integration-test -am
```
+
Notice that, this above maven command only run IT.
--------
+And if you want to run IT in the IDE like IntelliJ, you need to achieve the effect as the `OneCopyIT` profile in maven, which has already been set by default. Follow Steps 1-4 to achieve it.
-Run in IntelliJ in cluster mode is so easy,
-- Step 0. Optionally, when you run the test for the first time, or when you change the code of the module that the integration test module depends on, you may need to use the following command to generate `integration-test/target/template-node` for nodes of the pseudo cluster.
+- Step 0. Optionally, when you run the test for the first time, or when you change the code of the module that the integration test module depends on, you may need to use the following command to generate `integration-test/target/template-node` for the node to be tested.
+
```
-mvn clean package -DskipTests -pl integration-test -am -PClusterIT
+mvn clean package -DskipTests -pl integration-test -am
```
- Step 1. Run(Menu) -> Edit Configurations...
@@ -55,25 +56,26 @@ mvn clean package -DskipTests -pl integration-test -am -PClusterIT
- Step 3. Input some fields as the following picture
- 
+ 
-Integration Testing with Local Standalone Mode
--------------------
+## Integration Testing with Cluster Mode
-Integration testing with local standalone mode can be run with both maven and IDEs like IntelliJ.
+You can run the integration test in a 'real' cluster mode. At present, we have implemented a pseudo cluster with 1 config nodes and 3 data nodes.
+(As the test cases and the test environment are decoupled, we can easily implement other pseudo cluster or even a docker-based cluster later.)
The maven command is:
```
-mvn clean verify -DskipUTs -pl integration-test -am -PLocalStandaloneOnMppIT
+mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT
```
--------
-And if you want to run IT in the IDE like IntelliJ, you need to achieve the effect as the `LocalStandaloneOnMppIT` profile in maven. Follow Steps 1-4 to achieve it.
+If you want to run IT in `Cluster1` mode in the IDE like IntelliJ, you need to achieve the effect as the `ClusterIT` profile in maven explicitly. Follow Steps 1-4 to achieve it, which are almost the same as `OneCopy`.
-- Step 0. Optionally, when you run the test for the first time, or when you change the code of the module that the integration test module depends on, you may need to use the following command to generate `integration-test/target/template-node` for the node of the local standalone.
-It has the same effect as step 0 of the cluster mode counterpart; these two command's generations are the same content.
+
+- Step 0. Optionally, when you run the test for the first time, or when you change the code of the module that the integration test module depends on, you may need to use the following command to generate `integration-test/target/template-node` for nodes of the pseudo cluster.
+
+ It has the same effect as step 0 of the `OneCopy` counterpart; these two commands' generations are the same content.
```
-mvn clean package -DskipTests -pl integration-test -am -PLocalStandaloneOnMppIT
+mvn clean package -DskipTests -pl integration-test
```
- Step 1. Run(Menu) -> Edit Configurations...
@@ -85,4 +87,4 @@ mvn clean package -DskipTests -pl integration-test -am -PLocalStandaloneOnMppIT
- Step 3. Input some fields as the following picture
- 
+ 
\ No newline at end of file
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index e5d8b9fe1a..a01c5c288d 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -209,11 +209,15 @@
</build>
<profiles>
<profile>
- <id>LocalStandaloneIT</id>
+ <id>OneCopyIT</id>
<properties>
<integrationTest.includedGroups>org.apache.iotdb.itbase.category.LocalStandaloneIT</integrationTest.includedGroups>
<integrationTest.excludedGroups/>
- <integrationTest.testEnv>Standalone</integrationTest.testEnv>
+ <integrationTest.testEnv>OneCopy</integrationTest.testEnv>
+ <integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
+ <integrationTest.randomSelectWriteNode>false</integrationTest.randomSelectWriteNode>
+ <integrationTest.readAndVerifyWithMultiNode>false</integrationTest.readAndVerifyWithMultiNode>
+ <integrationTest.parallelMode>classes</integrationTest.parallelMode>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
@@ -246,20 +250,5 @@
<activeByDefault>false</activeByDefault>
</activation>
</profile>
- <profile>
- <id>LocalStandaloneOnMppIT</id>
- <properties>
- <integrationTest.includedGroups>org.apache.iotdb.itbase.category.LocalStandaloneIT</integrationTest.includedGroups>
- <integrationTest.excludedGroups/>
- <integrationTest.testEnv>LocalStandaloneOnMpp</integrationTest.testEnv>
- <integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
- <integrationTest.randomSelectWriteNode>false</integrationTest.randomSelectWriteNode>
- <integrationTest.readAndVerifyWithMultiNode>false</integrationTest.readAndVerifyWithMultiNode>
- <integrationTest.parallelMode>classes</integrationTest.parallelMode>
- </properties>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- </profile>
</profiles>
</project>
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigFactory.java
index 22f48193b5..639a963dff 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigFactory.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigFactory.java
@@ -20,39 +20,24 @@ package org.apache.iotdb.it.env;
import org.apache.iotdb.itbase.env.BaseConfig;
-import java.lang.reflect.InvocationTargetException;
-
public class ConfigFactory {
private static BaseConfig config;
public static BaseConfig getConfig() {
if (config == null) {
- try {
- switch (System.getProperty("TestEnv", "Standalone")) {
- case "Standalone":
- config =
- (BaseConfig)
- Class.forName("org.apache.iotdb.db.it.env.StandaloneEnvConfig")
- .getDeclaredConstructor()
- .newInstance();
- break;
- case "LocalStandaloneOnMpp":
- case "Cluster1":
- config = new MppConfig();
- break;
- case "Remote":
- config = new RemoteServerConfig();
- break;
- default:
- throw new ClassNotFoundException("The Property class of TestEnv not found");
- }
- } catch (ClassNotFoundException
- | IllegalAccessException
- | InstantiationException
- | NoSuchMethodException
- | InvocationTargetException e) {
- e.printStackTrace();
- System.exit(-1);
+ EnvType env = EnvType.getSystemEnvType();
+ switch (env) {
+ case OneCopy:
+ case Cluster1:
+ config = new MppConfig();
+ break;
+ case Remote:
+ config = new RemoteServerConfig();
+ break;
+ default:
+ System.out.println("Unknown env type: " + env);
+ System.exit(-1);
+ break;
}
}
return config;
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
index 614a640180..b275ce7440 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
@@ -33,23 +33,23 @@ public class EnvFactory {
try {
Class.forName(Config.JDBC_DRIVER_NAME);
logger.debug(">>>>>>>" + System.getProperty("TestEnv"));
- switch (System.getProperty("TestEnv", "Standalone")) {
- case "Standalone":
- env = (BaseEnv) Class.forName("org.apache.iotdb.db.it.env.StandaloneEnv").newInstance();
+ EnvType envType = EnvType.getSystemEnvType();
+ switch (envType) {
+ case OneCopy:
+ env = new OneCopyEnv();
break;
- case "LocalStandaloneOnMpp":
- env = new StandaloneOnMppEnv();
- break;
- case "Cluster1":
+ case Cluster1:
env = new Cluster1Env();
break;
- case "Remote":
+ case Remote:
env = new RemoteServerEnv();
break;
default:
- throw new ClassNotFoundException("The Property class of TestEnv not found");
+ System.out.println("Unknown env type: " + envType);
+ System.exit(-1);
+ break;
}
- } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
+ } catch (ClassNotFoundException e) {
e.printStackTrace();
System.exit(-1);
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java
similarity index 69%
rename from integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java
rename to integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java
index 3c245f3ae5..e88d958069 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java
@@ -18,15 +18,13 @@
*/
package org.apache.iotdb.it.env;
-public class StandaloneDataNodeWrapper extends DataNodeWrapper {
+public enum EnvType {
+ Remote,
+ OneCopy,
+ Cluster1;
- public StandaloneDataNodeWrapper(
- String targetConfigNode, String testClassName, String testMethodName, int[] portList) {
- super(targetConfigNode, testClassName, testMethodName, portList);
- }
-
- @Override
- protected String mainClassName() {
- return "org.apache.iotdb.db.service.NewIoTDB";
+ public static EnvType getSystemEnvType() {
+ String envValue = System.getProperty("TestEnv", OneCopy.name());
+ return EnvType.valueOf(envValue);
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/OneCopyEnv.java
similarity index 67%
rename from integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java
rename to integration-test/src/main/java/org/apache/iotdb/it/env/OneCopyEnv.java
index d770a5b464..bf438ce882 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/OneCopyEnv.java
@@ -22,30 +22,14 @@ import org.apache.iotdb.it.framework.IoTDBTestLogger;
import org.slf4j.Logger;
-import java.util.Collections;
-
-public class StandaloneOnMppEnv extends AbstractEnv {
+public class OneCopyEnv extends AbstractEnv {
private static final Logger logger = IoTDBTestLogger.logger;
- private void initEnvironment() {
- DataNodeWrapper dataNodeWrapper =
- new StandaloneDataNodeWrapper(
- null,
- super.getTestClassName(),
- super.getTestMethodName(),
- EnvUtils.searchAvailablePorts());
- dataNodeWrapper.createDir();
- dataNodeWrapper.changeConfig(ConfigFactory.getConfig().getEngineProperties());
- dataNodeWrapper.start();
- super.dataNodeWrapperList = Collections.singletonList(dataNodeWrapper);
- super.testWorking();
- }
-
@Override
public void initBeforeClass() throws InterruptedException {
logger.debug("=======start init class=======");
- initEnvironment();
+ super.initEnvironment(1, 1);
}
@Override
@@ -56,6 +40,6 @@ public class StandaloneOnMppEnv extends AbstractEnv {
@Override
public void initBeforeTest() {
logger.debug("=======start init test=======");
- initEnvironment();
+ initEnvironment(1, 1);
}
}