You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2021/05/02 04:26:19 UTC
[iotdb] branch test_container created (now 7007228)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch test_container
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 7007228 refact the testcontainer to cover more cluster cases
This branch includes the following new commits:
new 7007228 refact the testcontainer to cover more cluster cases
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: refact the testcontainer to cover more cluster cases
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch test_container
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 70072281530febcecc902a5bf3c073be1275fba7
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Sun May 2 12:25:35 2021 +0800
refact the testcontainer to cover more cluster cases
---
.../java/org/apache/iotdb/db/sql/ClusterIT.java | 102 +++++++++++++--------
.../AbstractThreeNodeClusterIT.java} | 62 +++----------
.../iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java | 4 +
.../iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java | 9 ++
.../AbstractFiveNodeClusterIT.java} | 70 ++++----------
.../iotdb/db/sql/nodes5/FiveNodeCluster1IT.java | 4 +
.../iotdb/db/sql/nodes5/FiveNodeCluster2IT.java | 12 +++
.../iotdb/db/sql/nodes5/FiveNodeCluster4IT.java | 9 ++
.../test/resources/1node/iotdb-cluster.properties | 2 +-
9 files changed, 132 insertions(+), 142 deletions(-)
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
index 72dd62b..ce6a681 100644
--- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
@@ -16,63 +16,77 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.sql;
import org.apache.iotdb.jdbc.Config;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.DockerComposeContainer;
-import org.testcontainers.containers.NoProjectNameDockerComposeContainer;
-import org.testcontainers.containers.output.Slf4jLogConsumer;
-import org.testcontainers.containers.wait.strategy.Wait;
-import java.io.File;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
-public class ClusterIT {
+public abstract class ClusterIT {
+
private static Logger logger = LoggerFactory.getLogger(ClusterIT.class);
- private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1");
- private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2");
- private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3");
-
- private Statement statement;
- private Connection connection;
-
- // in TestContainer's document, it is @ClassRule, and the environment is `public static`
- // I am not sure the difference now.
- @Rule
- public DockerComposeContainer environment =
- new NoProjectNameDockerComposeContainer(
- "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml"))
- .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort())
- .withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger))
- .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort())
- .withLogConsumer("iotdb-server_2", new Slf4jLogConsumer(node2Logger))
- .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort())
- .withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger))
- .withLocalCompose(true);
-
- int rpcPort = 6667;
+
+ private Statement writeStatement;
+ private Connection writeConnection;
+ private Statement readStatement;
+ private Connection readConnection;
+
+ protected int getWriteRpcPort() {
+ return getContainer().getServicePort("iotdb-server_1", 6667);
+ }
+
+ protected String getWriteRpcIp() {
+ return getContainer().getServiceHost("iotdb-server_1", 6667);
+ }
+
+ protected int getReadRpcPort() {
+ return getContainer().getServicePort("iotdb-server_1", 6667);
+ }
+
+ protected String getReadRpcIp() {
+ return getContainer().getServiceHost("iotdb-server_1", 6667);
+ }
+
+ protected void startCluster() {}
+
+ protected abstract DockerComposeContainer getContainer();
@Before
public void setUp() throws Exception {
-
- String ip = environment.getServiceHost("iotdb-server_1", 6667);
- rpcPort = environment.getServicePort("iotdb-server_1", 6667);
+ startCluster();
Class.forName(Config.JDBC_DRIVER_NAME);
- connection = DriverManager.getConnection("jdbc:iotdb://" + ip + ":" + rpcPort, "root", "root");
- statement = connection.createStatement();
+ writeConnection =
+ DriverManager.getConnection(
+ "jdbc:iotdb://" + getWriteRpcIp() + ":" + getWriteRpcPort(), "root", "root");
+ writeStatement = writeConnection.createStatement();
+ readConnection =
+ DriverManager.getConnection(
+ "jdbc:iotdb://" + getReadRpcIp() + ":" + getReadRpcPort(), "root", "root");
+ readStatement = readConnection.createStatement();
}
@After
public void tearDown() throws Exception {
- statement.close();
- connection.close();
+ writeStatement.close();
+ writeConnection.close();
+ readStatement.close();
+ readConnection.close();
}
@Test
@@ -81,13 +95,13 @@ public class ClusterIT {
String[] timeSeriesArray = {"root.sg1.aa.bb", "root.sg1.aa.bb.cc", "root.sg1.aa"};
for (String timeSeries : timeSeriesArray) {
- statement.execute(
+ writeStatement.execute(
String.format(
"create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY",
timeSeries));
}
ResultSet resultSet = null;
- resultSet = statement.executeQuery("show timeseries");
+ resultSet = readStatement.executeQuery("show timeseries");
Set<String> result = new HashSet<>();
while (resultSet.next()) {
result.add(resultSet.getString(1));
@@ -96,5 +110,17 @@ public class ClusterIT {
for (String timeseries : timeSeriesArray) {
Assert.assertTrue(result.contains(timeseries));
}
+ resultSet.close();
+
+ // test https://issues.apache.org/jira/browse/IOTDB-1331
+ writeStatement.execute("insert into root.ln.wf01.wt01(time, temperature) values(10, 1.0)");
+ resultSet = readStatement.executeQuery("select avg(temperature) from root.ln.wf01.wt01");
+ if (resultSet.next()) {
+ Assert.assertEquals(1.0, resultSet.getDouble(1), 0.01);
+ } else {
+ Assert.fail("expect 1 result, but get an empty resultSet.");
+ }
+ Assert.assertFalse(resultSet.next());
+ resultSet.close();
}
}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java
similarity index 58%
copy from testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
copy to testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java
index 72dd62b..059b4b0 100644
--- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.sql;
-import org.apache.iotdb.jdbc.Config;
+package org.apache.iotdb.db.sql.nodes3;
-import org.junit.*;
+import org.apache.iotdb.db.sql.ClusterIT;
+import org.junit.Rule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.DockerComposeContainer;
@@ -29,19 +29,16 @@ import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import java.io.File;
-import java.sql.*;
-import java.util.HashSet;
-import java.util.Set;
-public class ClusterIT {
- private static Logger logger = LoggerFactory.getLogger(ClusterIT.class);
+// just add real ITs into AbstractClusterIT.
+// in this case, the data must be on all nodes.
+// we just simulate write data on node A and read data on either node A or B.
+public abstract class AbstractThreeNodeClusterIT extends ClusterIT {
+
private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1");
private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2");
private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3");
- private Statement statement;
- private Connection connection;
-
// in TestContainer's document, it is @ClassRule, and the environment is `public static`
// I am not sure the difference now.
@Rule
@@ -56,45 +53,8 @@ public class ClusterIT {
.withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger))
.withLocalCompose(true);
- int rpcPort = 6667;
-
- @Before
- public void setUp() throws Exception {
-
- String ip = environment.getServiceHost("iotdb-server_1", 6667);
- rpcPort = environment.getServicePort("iotdb-server_1", 6667);
-
- Class.forName(Config.JDBC_DRIVER_NAME);
- connection = DriverManager.getConnection("jdbc:iotdb://" + ip + ":" + rpcPort, "root", "root");
- statement = connection.createStatement();
- }
-
- @After
- public void tearDown() throws Exception {
- statement.close();
- connection.close();
- }
-
- @Test
- public void testSimplePutAndGet() throws SQLException {
-
- String[] timeSeriesArray = {"root.sg1.aa.bb", "root.sg1.aa.bb.cc", "root.sg1.aa"};
-
- for (String timeSeries : timeSeriesArray) {
- statement.execute(
- String.format(
- "create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY",
- timeSeries));
- }
- ResultSet resultSet = null;
- resultSet = statement.executeQuery("show timeseries");
- Set<String> result = new HashSet<>();
- while (resultSet.next()) {
- result.add(resultSet.getString(1));
- }
- Assert.assertEquals(3, result.size());
- for (String timeseries : timeSeriesArray) {
- Assert.assertTrue(result.contains(timeseries));
- }
+ @Override
+ protected DockerComposeContainer getContainer() {
+ return environment;
}
}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java
new file mode 100644
index 0000000..eef2fd0
--- /dev/null
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java
@@ -0,0 +1,4 @@
+package org.apache.iotdb.db.sql.nodes3;
+
+// read and the write statements are on the same node.
+public class ThreeNodeCluster1IT extends AbstractThreeNodeClusterIT {}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java
new file mode 100644
index 0000000..1a5134a
--- /dev/null
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java
@@ -0,0 +1,9 @@
+package org.apache.iotdb.db.sql.nodes3;
+
+// read and the write statements are on the different nodes.
+public class ThreeNodeCluster2IT extends AbstractThreeNodeClusterIT {
+
+ protected String getWriteRpcIp() {
+ return getContainer().getServiceHost("iotdb-server_2", 6667);
+ }
+}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java
similarity index 56%
copy from testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
copy to testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java
index 72dd62b..126c3f6 100644
--- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.sql;
-import org.apache.iotdb.jdbc.Config;
+package org.apache.iotdb.db.sql.nodes5;
-import org.junit.*;
+import org.apache.iotdb.db.sql.ClusterIT;
+import org.junit.Rule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.DockerComposeContainer;
@@ -29,72 +29,38 @@ import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import java.io.File;
-import java.sql.*;
-import java.util.HashSet;
-import java.util.Set;
-public class ClusterIT {
- private static Logger logger = LoggerFactory.getLogger(ClusterIT.class);
+// just add real ITs into AbstractClusterIT.
+// in this case, the data may be not on the node that a read client connects.
+// So, we have: write data on node A and read data on either node A, B, and C.
+public abstract class AbstractFiveNodeClusterIT extends ClusterIT {
+
private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1");
private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2");
private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3");
-
- private Statement statement;
- private Connection connection;
+ private static Logger node4Logger = LoggerFactory.getLogger("iotdb-server_4");
+ private static Logger node5Logger = LoggerFactory.getLogger("iotdb-server_5");
// in TestContainer's document, it is @ClassRule, and the environment is `public static`
// I am not sure the difference now.
@Rule
public DockerComposeContainer environment =
new NoProjectNameDockerComposeContainer(
- "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml"))
+ "5nodes", new File("src/test/resources/5nodes/docker-compose.yaml"))
.withExposedService("iotdb-server_1", 6667, Wait.forListeningPort())
.withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger))
.withExposedService("iotdb-server_2", 6667, Wait.forListeningPort())
.withLogConsumer("iotdb-server_2", new Slf4jLogConsumer(node2Logger))
.withExposedService("iotdb-server_3", 6667, Wait.forListeningPort())
.withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger))
+ .withExposedService("iotdb-server_4", 6667, Wait.forListeningPort())
+ .withLogConsumer("iotdb-server_4", new Slf4jLogConsumer(node4Logger))
+ .withExposedService("iotdb-server_5", 6667, Wait.forListeningPort())
+ .withLogConsumer("iotdb-server_5", new Slf4jLogConsumer(node5Logger))
.withLocalCompose(true);
- int rpcPort = 6667;
-
- @Before
- public void setUp() throws Exception {
-
- String ip = environment.getServiceHost("iotdb-server_1", 6667);
- rpcPort = environment.getServicePort("iotdb-server_1", 6667);
-
- Class.forName(Config.JDBC_DRIVER_NAME);
- connection = DriverManager.getConnection("jdbc:iotdb://" + ip + ":" + rpcPort, "root", "root");
- statement = connection.createStatement();
- }
-
- @After
- public void tearDown() throws Exception {
- statement.close();
- connection.close();
- }
-
- @Test
- public void testSimplePutAndGet() throws SQLException {
-
- String[] timeSeriesArray = {"root.sg1.aa.bb", "root.sg1.aa.bb.cc", "root.sg1.aa"};
-
- for (String timeSeries : timeSeriesArray) {
- statement.execute(
- String.format(
- "create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY",
- timeSeries));
- }
- ResultSet resultSet = null;
- resultSet = statement.executeQuery("show timeseries");
- Set<String> result = new HashSet<>();
- while (resultSet.next()) {
- result.add(resultSet.getString(1));
- }
- Assert.assertEquals(3, result.size());
- for (String timeseries : timeSeriesArray) {
- Assert.assertTrue(result.contains(timeseries));
- }
+ @Override
+ protected DockerComposeContainer getContainer() {
+ return environment;
}
}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java
new file mode 100644
index 0000000..955d6f8
--- /dev/null
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java
@@ -0,0 +1,4 @@
+package org.apache.iotdb.db.sql.nodes5;
+
+// read and the write statements are on the different nodes.
+public class FiveNodeCluster1IT extends AbstractFiveNodeClusterIT {}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java
new file mode 100644
index 0000000..63e7a82
--- /dev/null
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java
@@ -0,0 +1,12 @@
+package org.apache.iotdb.db.sql.nodes5;
+
+import org.apache.iotdb.db.sql.nodes3.AbstractThreeNodeClusterIT;
+import org.testcontainers.containers.DockerComposeContainer;
+
+// read and the write statements are on the different nodes, and maybe in the same raft group.
+public class FiveNodeCluster2IT extends AbstractFiveNodeClusterIT {
+
+ protected String getWriteRpcIp() {
+ return getContainer().getServiceHost("iotdb-server_2", 6667);
+ }
+}
diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java
new file mode 100644
index 0000000..d3cfd82
--- /dev/null
+++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java
@@ -0,0 +1,9 @@
+package org.apache.iotdb.db.sql.nodes5;
+
+// read and the write statements are on the different nodes, and maybe in the different raft groups.
+public class FiveNodeCluster4IT extends AbstractFiveNodeClusterIT {
+
+ protected String getWriteRpcIp() {
+ return getContainer().getServiceHost("iotdb-server_4", 6667);
+ }
+}
diff --git a/testcontainer/src/test/resources/1node/iotdb-cluster.properties b/testcontainer/src/test/resources/1node/iotdb-cluster.properties
index 2df52cc..e60395a 100644
--- a/testcontainer/src/test/resources/1node/iotdb-cluster.properties
+++ b/testcontainer/src/test/resources/1node/iotdb-cluster.properties
@@ -18,7 +18,7 @@
internal_meta_port=9003
internal_data_port=40010
-seed_nodes=3nodes_iotdb-server_1:9003
+seed_nodes=1node_iotdb-server_1:9003
default_replica_num=1
consistency_level=mid
connection_timeout_ms=20000