You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by er...@apache.org on 2022/06/23 03:34:55 UTC
[iotdb] branch master updated: [IOTDB-3558] Dump node stacks if IT test request timeout (#6382)
This is an automated email from the ASF dual-hosted git repository.
ericpai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 074b37f5fe [IOTDB-3558] Dump node stacks if IT test request timeout (#6382)
074b37f5fe is described below
commit 074b37f5fed8686854d7937f7de6a3b8bde4b891
Author: BaiJian <er...@hotmail.com>
AuthorDate: Thu Jun 23 11:34:49 2022 +0800
[IOTDB-3558] Dump node stacks if IT test request timeout (#6382)
---
integration-test/pom.xml | 2 +-
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 145 +++++++++-----
.../apache/iotdb/it/env/AbstractNodeWrapper.java | 223 ++++++++++++++-------
.../org/apache/iotdb/it/env/ConfigNodeWrapper.java | 49 ++---
.../org/apache/iotdb/it/env/DataNodeWrapper.java | 35 ++--
.../org/apache/iotdb/it/env/IoTDBTestRunner.java | 2 +-
.../org/apache/iotdb/it/env/RemoteServerEnv.java | 7 +-
.../iotdb/it/env/StandaloneDataNodeWrapper.java | 5 +-
.../apache/iotdb/it/env/StandaloneOnMppEnv.java | 34 +---
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 4 +-
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 4 +
.../itbase/runtime/InconsistentDataException.java | 2 +-
.../itbase/runtime/ParallelRequestDelegate.java | 3 +
.../iotdb/itbase/runtime/RequestDelegate.java | 5 +-
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 6 +-
15 files changed, 315 insertions(+), 211 deletions(-)
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 4bd7f56894..4ba31b5ffd 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -190,7 +190,7 @@
<excludedGroups>${integrationTest.excludedGroups}</excludedGroups>
<useSystemClassLoader>false</useSystemClassLoader>
<parallel>${integrationTest.parallelMode}</parallel>
- <useUnlimitedThreads>true</useUnlimitedThreads>
+ <threadCount>1</threadCount>
<forkCount>${integrationTest.forkCount}</forkCount>
<reuseForks>false</reuseForks>
<systemPropertyVariables>
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractEnv.java
index c41ea0c432..88db28bd78 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractEnv.java
@@ -28,19 +28,23 @@ import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.Constant;
import org.apache.iotdb.jdbc.IoTDBConnection;
+import org.apache.commons.lang3.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Date;
+import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
import static org.apache.iotdb.jdbc.Config.VERSION;
import static org.junit.Assert.fail;
@@ -50,31 +54,34 @@ public abstract class AbstractEnv implements BaseEnv {
private final int NODE_START_TIMEOUT = 100;
private final int PROBE_TIMEOUT_MS = 2000;
private final int NODE_NETWORK_TIMEOUT_MS = 65_000;
- protected List<ConfigNodeWrapper> configNodeWrapperList;
- protected List<DataNodeWrapper> dataNodeWrapperList;
+ private final String lockFilePath =
+ System.getProperty("user.dir") + File.separator + "target" + File.separator + "lock-";
+ protected List<ConfigNodeWrapper> configNodeWrapperList = Collections.emptyList();
+ protected List<DataNodeWrapper> dataNodeWrapperList = Collections.emptyList();
private final Random rand = new Random();
- protected String nextTestCase = null;
+ protected String testMethodName = null;
- protected void initEnvironment(int configNodesNum, int dataNodesNum) throws InterruptedException {
+ protected void initEnvironment(int configNodesNum, int dataNodesNum) {
this.configNodeWrapperList = new ArrayList<>();
this.dataNodeWrapperList = new ArrayList<>();
- final String testName = getTestClassName() + getNextTestCaseString();
+ final String testClassName = getTestClassName();
+ final String testMethodName = getTestMethodName();
- ConfigNodeWrapper seedConfigNodeWrapper = new ConfigNodeWrapper(true, "", testName);
+ ConfigNodeWrapper seedConfigNodeWrapper =
+ new ConfigNodeWrapper(true, "", testClassName, testMethodName, searchAvailablePorts());
seedConfigNodeWrapper.createDir();
seedConfigNodeWrapper.changeConfig(ConfigFactory.getConfig().getConfignodeProperties());
seedConfigNodeWrapper.start();
String targetConfigNode = seedConfigNodeWrapper.getIpAndPortString();
this.configNodeWrapperList.add(seedConfigNodeWrapper);
- logger.info(
- "In test " + testName + " SeedConfigNode " + seedConfigNodeWrapper.getId() + " started.");
List<String> configNodeEndpoints = new ArrayList<>();
RequestDelegate<Void> configNodesDelegate = new SerialRequestDelegate<>(configNodeEndpoints);
for (int i = 1; i < configNodesNum; i++) {
ConfigNodeWrapper configNodeWrapper =
- new ConfigNodeWrapper(false, targetConfigNode, testName);
+ new ConfigNodeWrapper(
+ false, targetConfigNode, testClassName, testMethodName, searchAvailablePorts());
this.configNodeWrapperList.add(configNodeWrapper);
configNodeEndpoints.add(configNodeWrapper.getIpAndPortString());
configNodeWrapper.createDir();
@@ -82,8 +89,6 @@ public abstract class AbstractEnv implements BaseEnv {
configNodesDelegate.addRequest(
() -> {
configNodeWrapper.start();
- logger.info(
- "In test " + testName + " ConfigNode " + configNodeWrapper.getId() + " started.");
return null;
});
}
@@ -98,7 +103,9 @@ public abstract class AbstractEnv implements BaseEnv {
RequestDelegate<Void> dataNodesDelegate =
new ParallelRequestDelegate<>(dataNodeEndpoints, NODE_START_TIMEOUT);
for (int i = 0; i < dataNodesNum; i++) {
- DataNodeWrapper dataNodeWrapper = new DataNodeWrapper(targetConfigNode, testName);
+ DataNodeWrapper dataNodeWrapper =
+ new DataNodeWrapper(
+ targetConfigNode, testClassName, testMethodName, searchAvailablePorts());
this.dataNodeWrapperList.add(dataNodeWrapper);
dataNodeEndpoints.add(dataNodeWrapper.getIpAndPortString());
dataNodeWrapper.createDir();
@@ -106,8 +113,6 @@ public abstract class AbstractEnv implements BaseEnv {
dataNodesDelegate.addRequest(
() -> {
dataNodeWrapper.start();
- logger.info(
- "In test " + testName + " DataNode " + dataNodeWrapper.getId() + " started.");
return null;
});
}
@@ -123,17 +128,18 @@ public abstract class AbstractEnv implements BaseEnv {
}
private void cleanupEnvironment() {
- for (DataNodeWrapper dataNodeWrapper : this.dataNodeWrapperList) {
- dataNodeWrapper.stop();
- dataNodeWrapper.waitingToShutDown();
- dataNodeWrapper.destroyDir();
- }
- for (ConfigNodeWrapper configNodeWrapper : this.configNodeWrapperList) {
- configNodeWrapper.stop();
- configNodeWrapper.waitingToShutDown();
- configNodeWrapper.destroyDir();
+ for (AbstractNodeWrapper nodeWrapper :
+ Stream.concat(this.dataNodeWrapperList.stream(), this.configNodeWrapperList.stream())
+ .collect(Collectors.toList())) {
+ nodeWrapper.stop();
+ nodeWrapper.waitingToShutDown();
+ nodeWrapper.destroyDir();
+ String lockPath = getLockFilePath(nodeWrapper.getPort());
+ if (!new File(lockPath).delete()) {
+ logger.error("Delete lock file {} failed", lockPath);
+ }
}
- nextTestCase = null;
+ testMethodName = null;
}
public String getTestClassName() {
@@ -147,19 +153,6 @@ public abstract class AbstractEnv implements BaseEnv {
return "UNKNOWN-IT";
}
- public String getTestClassNameAndDate() {
- Date date = new Date();
- SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HHmmss");
- StackTraceElement[] stack = Thread.currentThread().getStackTrace();
- for (StackTraceElement stackTraceElement : stack) {
- String className = stackTraceElement.getClassName();
- if (className.endsWith("IT")) {
- return className.substring(className.lastIndexOf(".") + 1) + "-" + formatter.format(date);
- }
- }
- return "IT";
- }
-
public void testWorking() {
List<String> endpoints =
dataNodeWrapperList.stream()
@@ -200,14 +193,55 @@ public abstract class AbstractEnv implements BaseEnv {
cleanupEnvironment();
}
- @Override
- public Connection getConnection() throws SQLException {
- return new ClusterTestConnection(getWriteConnection(null), getReadConnections(null));
+ public final int[] searchAvailablePorts() {
+ do {
+ int randomPortStart = 1000 + (int) (Math.random() * (1999 - 1000));
+ randomPortStart = randomPortStart * 10 + 1;
+ File lockFile = new File(getLockFilePath(randomPortStart));
+ if (lockFile.exists()) {
+ continue;
+ }
+
+ List<Integer> requiredPorts =
+ IntStream.rangeClosed(randomPortStart, randomPortStart + 9)
+ .boxed()
+ .collect(Collectors.toList());
+ String cmd = getSearchAvailablePortCmd(requiredPorts);
+
+ try {
+ Process proc = Runtime.getRuntime().exec(cmd);
+ if (proc.waitFor() == 1 && lockFile.createNewFile()) {
+ return requiredPorts.stream().mapToInt(Integer::intValue).toArray();
+ }
+ } catch (IOException e) {
+ // ignore
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ } while (true);
+ }
+
+ private String getSearchAvailablePortCmd(List<Integer> ports) {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ return getWindowsSearchPortCmd(ports);
+ }
+ return getUnixSearchPortCmd(ports);
+ }
+
+ private String getWindowsSearchPortCmd(List<Integer> ports) {
+ String cmd = "netstat -aon -p tcp | findStr ";
+ return cmd
+ + ports.stream().map(v -> "/C:'127.0.0.1:" + v + "'").collect(Collectors.joining(" "));
+ }
+
+ private String getUnixSearchPortCmd(List<Integer> ports) {
+ String cmd = "lsof -iTCP -sTCP:LISTEN -P -n | awk '{print $9}' | grep -E ";
+ return cmd + ports.stream().map(String::valueOf).collect(Collectors.joining("|")) + "\"";
}
@Override
- public void setNextTestCaseName(String testCaseName) {
- nextTestCase = testCaseName;
+ public Connection getConnection() throws SQLException {
+ return new ClusterTestConnection(getWriteConnection(null), getReadConnections(null));
}
private Connection getConnection(String endpoint, int queryTimeout) throws SQLException {
@@ -287,10 +321,25 @@ public abstract class AbstractEnv implements BaseEnv {
return sb.toString();
}
- public String getNextTestCaseString() {
- if (nextTestCase != null) {
- return "_" + nextTestCase;
+ public String getTestMethodName() {
+ return testMethodName;
+ }
+
+ @Override
+ public void setTestMethodName(String testMethodName) {
+ this.testMethodName = testMethodName;
+ }
+
+ public void dumpTestJVMSnapshot() {
+ for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) {
+ configNodeWrapper.dumpJVMSnapshot(testMethodName);
+ }
+ for (DataNodeWrapper dataNodeWrapper : dataNodeWrapperList) {
+ dataNodeWrapper.dumpJVMSnapshot(testMethodName);
}
- return "";
+ }
+
+ private String getLockFilePath(int port) {
+ return lockFilePath + port;
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java
index c2390690b0..3cd791767c 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java
@@ -25,19 +25,32 @@ import org.apache.commons.lang3.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
+import javax.management.MBeanServerConnection;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.net.MalformedURLException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.List;
+import java.time.LocalDateTime;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import static org.junit.Assert.fail;
@@ -47,78 +60,33 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
System.getProperty("user.dir") + File.separator + "target" + File.separator + "template-node";
private final File NULL_FILE =
SystemUtils.IS_OS_WINDOWS ? new File("nul") : new File("/dev/null");
-
+ protected final String testClassName;
+ protected final String testMethodName;
+ protected final int[] portList;
+ private final int jmxPort;
+ private final String jmxUserName = "root";
+ private final String jmxPassword = "passw!d";
+ private final String TAB = " ";
private Process instance;
- protected final String testName;
-
- public AbstractNodeWrapper(String testName) {
- this.testName = testName;
- }
-
- protected final String getId() {
- return "node" + getPort();
- }
-
- protected final int[] searchAvailablePorts() {
- do {
- int randomPortStart = 1000 + (int) (Math.random() * (1999 - 1000));
- randomPortStart = randomPortStart * 10 + 1;
- String path =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "node"
- + randomPortStart;
- if (new File(path).exists()) {
- continue;
- }
-
- List<Integer> requiredPorts =
- IntStream.rangeClosed(randomPortStart, randomPortStart + 9)
- .boxed()
- .collect(Collectors.toList());
- String cmd = getSearchAvailablePortCmd(requiredPorts);
- try {
- Process proc = Runtime.getRuntime().exec(cmd);
- BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()));
- String line;
- while ((line = br.readLine()) != null) {
- logger.debug(line);
- }
- if (proc.waitFor() == 1) {
- return requiredPorts.stream().mapToInt(Integer::intValue).toArray();
- }
- } catch (IOException | InterruptedException ex) {
- // ignore
- }
- } while (true);
- }
-
- private String getSearchAvailablePortCmd(List<Integer> ports) {
- if (SystemUtils.IS_OS_WINDOWS) {
- return getWindowsSearchPortCmd(ports);
- }
- return getUnixSearchPortCmd(ports);
- }
-
- private String getWindowsSearchPortCmd(List<Integer> ports) {
- String cmd = "netstat -aon -p tcp | findStr ";
- return cmd
- + ports.stream().map(v -> "/C:'127.0.0.1:" + v + "'").collect(Collectors.joining(" "));
- }
-
- private String getUnixSearchPortCmd(List<Integer> ports) {
- String cmd = "lsof -iTCP -sTCP:LISTEN -P -n | awk '{print $9}' | grep -E ";
- return cmd + ports.stream().map(String::valueOf).collect(Collectors.joining("|")) + "\"";
+ public AbstractNodeWrapper(String testClassName, String testMethodName, int[] portList) {
+ this.testClassName = testClassName;
+ this.testMethodName = testMethodName;
+ this.portList = portList;
+ jmxPort = this.portList[portList.length - 1];
}
@Override
public void createDir() {
// Copy templateNodePath to nodePath
try {
- FileUtils.copyDirectoryToDirectory(new File(this.templateNodePath), new File(getNodePath()));
+ File nodeDir = new File(getNodePath());
+ try {
+ FileUtils.forceDelete(nodeDir);
+ } catch (FileNotFoundException e) {
+ // ignored
+ }
+ FileUtils.copyDirectoryToDirectory(new File(this.templateNodePath), nodeDir);
String startScriptPath = getStartScriptPath();
String stopScriptPath = getStopScriptPath();
if (!new File(startScriptPath).setExecutable(true)) {
@@ -141,7 +109,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
FileUtils.forceDelete(new File(getNodePath()));
return;
} catch (IOException ex) {
- logger.error("Delete node dir failed. RetryTimes={}", i + 1, ex);
+ logger.warn("Delete node dir failed. RetryTimes={}", i + 1, ex);
try {
TimeUnit.SECONDS.sleep(3);
} catch (InterruptedException e) {
@@ -161,6 +129,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
.redirectOutput(stdoutFile)
.redirectError(stdoutFile);
this.instance = processBuilder.start();
+ logger.info("In test {} {} started.", getTestLogDirName(), getId());
} catch (IOException ex) {
fail("Start node failed. " + ex);
}
@@ -219,6 +188,12 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
try (FileWriter confOutput = new FileWriter(configPath)) {
configProperties.store(confOutput, null);
}
+ // Change JMX config
+ Path path = Paths.get(getEnvConfigPath());
+ String content = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
+ content = content.replaceAll("JMX_LOCAL=\"true\"", "JMX_LOCAL=\"false\"");
+ content = content.replaceAll("JMX_PORT=\"\\d+\"", String.format("JMX_PORT=\"%d\"", jmxPort));
+ Files.write(path, content.getBytes(StandardCharsets.UTF_8));
} catch (IOException ex) {
fail("Change the config of data node failed. " + ex);
}
@@ -229,6 +204,11 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
return "127.0.0.1";
}
+ @Override
+ public final int getPort() {
+ return portList[0];
+ }
+
@Override
public final String getIpAndPortString() {
return this.getIp() + ":" + this.getPort();
@@ -246,15 +226,114 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
protected abstract String getConfigPath();
+ protected abstract String getEnvConfigPath();
+
protected abstract void updateConfig(Properties properties);
protected abstract String getStartScriptPath();
protected abstract String getStopScriptPath();
- protected abstract String getLogPath();
+ private String getLogPath() {
+ return getLogDirPath() + File.separator + getId() + ".log";
+ }
- protected String getNodePath() {
+ private String getLogDirPath() {
+ return System.getProperty("user.dir")
+ + File.separator
+ + "target"
+ + File.separator
+ + "cluster-logs"
+ + File.separator
+ + getTestLogDirName();
+ }
+
+ private String getNodePath() {
return System.getProperty("user.dir") + File.separator + "target" + File.separator + getId();
}
+
+ @Override
+ public void dumpJVMSnapshot(String testCaseName) {
+ JMXServiceURL url;
+ try {
+ url =
+ new JMXServiceURL(
+ String.format("service:jmx:rmi:///jndi/rmi://127.0.0.1:%d/jmxrmi", jmxPort));
+ } catch (MalformedURLException e) {
+ logger.error("Construct JMX URL failed", e);
+ return;
+ }
+ Map<String, Object> environment =
+ Collections.singletonMap(JMXConnector.CREDENTIALS, new String[] {jmxUserName, jmxPassword});
+
+ try (JMXConnector connector = JMXConnectorFactory.connect(url, environment)) {
+ MBeanServerConnection mbsc = connector.getMBeanServerConnection();
+ ThreadMXBean tmbean =
+ ManagementFactory.newPlatformMXBeanProxy(
+ mbsc, ManagementFactory.THREAD_MXBEAN_NAME, ThreadMXBean.class);
+ ThreadInfo[] threadInfos = tmbean.dumpAllThreads(true, true);
+ long[] deadlockIds = tmbean.findDeadlockedThreads();
+ LocalDateTime currentTime = LocalDateTime.now();
+ try (PrintWriter output =
+ new PrintWriter(
+ getLogDirPath() + File.separator + testCaseName + "_" + getId() + "-threads.dump")) {
+ output.printf("# Captured at %s\n", currentTime);
+ output.println("==================\n");
+ if (deadlockIds != null && deadlockIds.length > 0) {
+ output.printf("Detect DEADLOCK threads!\n");
+ for (long deadlockId : deadlockIds) {
+ ThreadInfo ti = tmbean.getThreadInfo(deadlockId);
+ output.printf("%s #%d\n", ti.getThreadName(), ti.getThreadId());
+ }
+ output.println("==================\n");
+ }
+ for (ThreadInfo threadInfo : threadInfos) {
+ dumpThread(output, threadInfo);
+ }
+ }
+ } catch (IOException e) {
+ logger.error("Connect with MBeanServer {} failed", url.getURLPath(), e);
+ }
+ }
+
+ private void dumpThread(PrintWriter output, ThreadInfo ti) {
+ StringBuilder sb = new StringBuilder("\"" + ti.getThreadName() + "\"");
+ sb.append(" #").append(ti.getThreadId());
+ sb.append(" ").append(ti.getThreadState()).append("\n");
+ if (ti.getLockName() != null) {
+ sb.append("Waiting on: ").append(ti.getLockName()).append("\n");
+ }
+ if (ti.getLockOwnerName() != null) {
+ sb.append("Locked by: ")
+ .append(ti.getLockOwnerName())
+ .append(" #")
+ .append(ti.getLockOwnerId())
+ .append("\n");
+ }
+ Map<StackTraceElement, MonitorInfo> lockInfoMap = new HashMap<>();
+ for (MonitorInfo monitor : ti.getLockedMonitors()) {
+ lockInfoMap.put(monitor.getLockedStackFrame(), monitor);
+ }
+ for (StackTraceElement ele : ti.getStackTrace()) {
+ if (lockInfoMap.containsKey(ele)) {
+ MonitorInfo monitorInfo = lockInfoMap.get(ele);
+ sb.append(TAB)
+ .append("- lock ")
+ .append(monitorInfo.getClassName())
+ .append("@")
+ .append(Integer.toHexString(monitorInfo.getIdentityHashCode()))
+ .append("\n");
+ }
+ sb.append(TAB).append(ele).append("\n");
+ }
+ sb.append("\n");
+ output.print(sb);
+ }
+
+ private String getTestLogDirName() {
+ if (testMethodName == null) {
+ return testClassName;
+ }
+ return testClassName + "_" + testMethodName;
+ }
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigNodeWrapper.java
index cd3ff3b4ee..e023455feb 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/ConfigNodeWrapper.java
@@ -27,14 +27,15 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper {
private final int consensusPort;
private final String targetConfigNode;
- private final int[] portList;
- public ConfigNodeWrapper(boolean isSeed, String targetConfigNode, String testName) {
- super(testName);
-
- portList = super.searchAvailablePorts();
+ public ConfigNodeWrapper(
+ boolean isSeed,
+ String targetConfigNode,
+ String testClassName,
+ String testMethodName,
+ int[] portList) {
+ super(testClassName, testMethodName, portList);
this.consensusPort = portList[1];
-
if (isSeed) {
this.targetConfigNode = getIpAndPortString();
} else {
@@ -67,6 +68,14 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper {
return workDirFilePath("confignode" + File.separator + "conf", "iotdb-confignode.properties");
}
+ @Override
+ protected String getEnvConfigPath() {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ return workDirFilePath("confignode" + File.separator + "conf", "confignode-env.bat");
+ }
+ return workDirFilePath("confignode" + File.separator + "conf", "confignode-env.sh");
+ }
+
@Override
protected String getStartScriptPath() {
String scriptName = "start-confignode.sh";
@@ -86,31 +95,7 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper {
}
@Override
- protected String getLogPath() {
- return System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "cluster-logs"
- + File.separator
- + testName
- + File.separator
- + "Config"
- + super.getId()
- + ".log";
- }
-
- @Override
- protected String getNodePath() {
- return System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + super.getId();
- }
-
- @Override
- public int getPort() {
- return portList[0];
+ public final String getId() {
+ return "ConfigNode" + getPort();
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/DataNodeWrapper.java
index 87779bd2be..6940839f52 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/DataNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/DataNodeWrapper.java
@@ -26,17 +26,15 @@ import java.util.Properties;
public class DataNodeWrapper extends AbstractNodeWrapper {
private final String targetConfigNode;
-
private final int dataBlockManagerPort;
private final int internalPort;
private final int dataRegionConsensusPort;
private final int schemaRegionConsensusPort;
- private final int[] portList;
- public DataNodeWrapper(String targetConfigNode, String testName) {
- super(testName);
+ public DataNodeWrapper(
+ String targetConfigNode, String testClassName, String testMethodName, int[] portList) {
+ super(testClassName, testMethodName, portList);
this.targetConfigNode = targetConfigNode;
- portList = super.searchAvailablePorts();
this.dataBlockManagerPort = portList[1];
this.internalPort = portList[2];
this.dataRegionConsensusPort = portList[3];
@@ -65,6 +63,14 @@ public class DataNodeWrapper extends AbstractNodeWrapper {
return workDirFilePath("datanode" + File.separator + "conf", "iotdb-engine.properties");
}
+ @Override
+ protected String getEnvConfigPath() {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ return workDirFilePath("datanode" + File.separator + "conf", "iotdb-env.bat");
+ }
+ return workDirFilePath("datanode" + File.separator + "conf", "iotdb-env.sh");
+ }
+
@Override
protected String getStartScriptPath() {
String scriptName = "start-datanode.sh";
@@ -84,22 +90,7 @@ public class DataNodeWrapper extends AbstractNodeWrapper {
}
@Override
- protected String getLogPath() {
- return System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "cluster-logs"
- + File.separator
- + testName
- + File.separator
- + "Data"
- + super.getId()
- + ".log";
- }
-
- @Override
- public int getPort() {
- return portList[0];
+ public final String getId() {
+ return "DataNode" + getPort();
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/IoTDBTestRunner.java b/integration-test/src/main/java/org/apache/iotdb/it/env/IoTDBTestRunner.java
index 433c7dc357..f160f8dfc4 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/IoTDBTestRunner.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/IoTDBTestRunner.java
@@ -38,7 +38,7 @@ public class IoTDBTestRunner extends BlockJUnit4ClassRunner {
protected void runChild(final FrameworkMethod method, RunNotifier notifier) {
Description description = describeChild(method);
logger.info("Run {}", description.getMethodName());
- EnvFactory.getEnv().setNextTestCaseName(description.getMethodName());
+ EnvFactory.getEnv().setTestMethodName(description.getMethodName());
super.runChild(method, notifier);
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/RemoteServerEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/RemoteServerEnv.java
index 4bf339f335..a9be62426f 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/RemoteServerEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/RemoteServerEnv.java
@@ -105,7 +105,12 @@ public class RemoteServerEnv implements BaseEnv {
return connection;
}
- public void setNextTestCaseName(String testCaseName) {
+ public void setTestMethodName(String testCaseName) {
+ // Do nothing
+ }
+
+ @Override
+ public void dumpTestJVMSnapshot() {
// Do nothing
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java
index 21ab80b13f..5bf6de27e8 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneDataNodeWrapper.java
@@ -24,8 +24,9 @@ import java.io.File;
public class StandaloneDataNodeWrapper extends DataNodeWrapper {
- public StandaloneDataNodeWrapper(String targetConfigNode, String testName) {
- super(targetConfigNode, testName);
+ public StandaloneDataNodeWrapper(
+ String targetConfigNode, String testClassName, String testMethodName, int[] portList) {
+ super(targetConfigNode, testClassName, testMethodName, portList);
}
@Override
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java
index 0d7e0fad35..810a03ba72 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/StandaloneOnMppEnv.java
@@ -21,21 +21,20 @@ package org.apache.iotdb.it.env;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
+import java.util.Collections;
public class StandaloneOnMppEnv extends AbstractEnv {
private static final Logger logger = LoggerFactory.getLogger(StandaloneOnMppEnv.class);
- private void initEnvironment() throws InterruptedException {
- super.dataNodeWrapperList = new ArrayList<>();
- final String testName = super.getTestClassName() + super.getNextTestCaseString();
- DataNodeWrapper dataNodeWrapper = new StandaloneDataNodeWrapper(null, testName);
+ private void initEnvironment() {
+ DataNodeWrapper dataNodeWrapper =
+ new StandaloneDataNodeWrapper(
+ null, super.getTestClassName(), super.getTestMethodName(), searchAvailablePorts());
dataNodeWrapper.createDir();
dataNodeWrapper.changeConfig(ConfigFactory.getConfig().getEngineProperties());
dataNodeWrapper.start();
- super.dataNodeWrapperList.add(dataNodeWrapper);
- logger.info("In test " + testName + " DataNode " + dataNodeWrapper.getId() + " started.");
+ super.dataNodeWrapperList = Collections.singletonList(dataNodeWrapper);
super.testWorking();
}
@@ -46,27 +45,8 @@ public class StandaloneOnMppEnv extends AbstractEnv {
}
@Override
- public void initBeforeTest() throws InterruptedException {
+ public void initBeforeTest() {
logger.debug("=======start init test=======");
initEnvironment();
}
-
- private void cleanupEnvironment() {
- for (DataNodeWrapper dataNodeWrapper : this.dataNodeWrapperList) {
- dataNodeWrapper.stop();
- dataNodeWrapper.waitingToShutDown();
- dataNodeWrapper.destroyDir();
- }
- super.nextTestCase = null;
- }
-
- @Override
- public void cleanAfterClass() {
- cleanupEnvironment();
- }
-
- @Override
- public void cleanAfterTest() {
- cleanupEnvironment();
- }
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java
index 5ed57e05b8..8e5650d44b 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java
@@ -37,5 +37,7 @@ public interface BaseEnv {
Connection getConnection(Constant.Version version) throws SQLException;
- void setNextTestCaseName(String testCaseName);
+ void setTestMethodName(String testCaseName);
+
+ void dumpTestJVMSnapshot();
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java
index ecb2e1fc05..c1efb1f351 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java
@@ -38,5 +38,9 @@ public interface BaseNodeWrapper {
int getPort();
+ String getId();
+
String getIpAndPortString();
+
+ void dumpJVMSnapshot(String testCaseName);
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/InconsistentDataException.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/InconsistentDataException.java
index 022f82120c..a6c49c242b 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/InconsistentDataException.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/InconsistentDataException.java
@@ -23,7 +23,7 @@ import java.util.List;
public class InconsistentDataException extends SQLException {
- public InconsistentDataException(Object data, List<String> endpoints) {
+ public <T> InconsistentDataException(List<T> data, List<String> endpoints) {
super(String.format("Datasets are inconsistent: %s with data %s", endpoints, data));
}
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ParallelRequestDelegate.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ParallelRequestDelegate.java
index 92b5b478bc..2fae21e56c 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ParallelRequestDelegate.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ParallelRequestDelegate.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.itbase.runtime;
+import org.apache.iotdb.it.env.EnvFactory;
+
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
@@ -53,6 +55,7 @@ public class ParallelRequestDelegate<T> extends RequestDelegate<T> {
} catch (ExecutionException e) {
exceptions[i] = e;
} catch (InterruptedException | TimeoutException e) {
+ EnvFactory.getEnv().dumpTestJVMSnapshot();
for (int j = i; j < getEndpoints().size(); j++) {
resultFutures.get(j).cancel(true);
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/RequestDelegate.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/RequestDelegate.java
index d191926fa1..1f5ffcf92b 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/RequestDelegate.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/RequestDelegate.java
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Callable;
@@ -107,7 +108,7 @@ public abstract class RequestDelegate<T> {
if (businessExceptions[i] != null) {
// As each exception has its own stacktrace, in order to display them clearly, we can only
// print them through logger.
- logger.error(
+ logger.warn(
"Exception happens during request to {}", getEndpoints().get(i), businessExceptions[i]);
}
}
@@ -115,7 +116,7 @@ public abstract class RequestDelegate<T> {
throw new SQLException(exceptionMsg[0]);
}
if (exceptionInconsistent) {
- throw new InconsistentDataException(exceptionMsg, getEndpoints());
+ throw new InconsistentDataException(Arrays.asList(exceptionMsg), getEndpoints());
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnv.java b/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnv.java
index efe6c728b2..dfc4d4fa9f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnv.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnv.java
@@ -87,7 +87,11 @@ public class StandaloneEnv implements BaseEnv {
return null;
}
- public void setNextTestCaseName(String testCaseName) {
+ public void setTestMethodName(String testCaseName) {
+ // Do nothing
+ }
+
+ public void dumpTestJVMSnapshot() {
// Do nothing
}
}