You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2024/09/02 11:01:18 UTC
(iotdb) 02/02: introduce AINode to IoTDB (fix)
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch addCIForAINode
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a42df558594aaf3a180c963caaa44690146a75d1
Author: ycycse <yc...@gmail.com>
AuthorDate: Mon Sep 2 18:59:33 2024 +0800
introduce AINode to IoTDB (fix)
---
.../java/org/apache/iotdb/it/env/EnvFactory.java | 2 +-
.../it/env/cluster/env/{MLEnv.java => AIEnv.java} | 0
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../{MLNodeWrapper.java => AINodeWrapper.java} | 166 ++++++---------------
.../it/env/cluster/node/AbstractNodeWrapper.java | 8 +-
.../{MLClusterIT.java => AIClusterIT.java} | 0
.../iotdb/mlnode/it/{MLEnvIT.java => AIEnvIT.java} | 0
7 files changed, 57 insertions(+), 132 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
index 88ac6f46f6e..ce8046708a8 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java
@@ -19,8 +19,8 @@
package org.apache.iotdb.it.env;
-import org.apache.iotdb.it.env.cluster.env.Cluster1Env;
import org.apache.iotdb.it.env.cluster.env.AIEnv;
+import org.apache.iotdb.it.env.cluster.env.Cluster1Env;
import org.apache.iotdb.it.env.cluster.env.SimpleEnv;
import org.apache.iotdb.it.env.remote.env.RemoteServerEnv;
import org.apache.iotdb.it.framework.IoTDBTestLogger;
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MLEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
similarity index 100%
rename from integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MLEnv.java
rename to integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java
index cb7a50c44c9..482e869cf1c 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java
@@ -41,10 +41,10 @@ import org.apache.iotdb.isession.pool.ISessionPool;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.config.*;
+import org.apache.iotdb.it.env.cluster.node.AINodeWrapper;
import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper;
import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
-import org.apache.iotdb.it.env.cluster.node.AINodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestLogger;
import org.apache.iotdb.itbase.env.BaseEnv;
import org.apache.iotdb.itbase.env.BaseNodeWrapper;
@@ -82,7 +82,7 @@ public abstract class AbstractEnv implements BaseEnv {
private final Random rand = new Random();
protected List<ConfigNodeWrapper> configNodeWrapperList = Collections.emptyList();
protected List<DataNodeWrapper> dataNodeWrapperList = Collections.emptyList();
- protected List<MLNodeWrapper> aiNodeWrapperList = Collections.emptyList();
+ protected List<AINodeWrapper> aiNodeWrapperList = Collections.emptyList();
protected String testMethodName = null;
protected int index = 0;
protected long startTime;
@@ -151,8 +151,8 @@ public abstract class AbstractEnv implements BaseEnv {
}
protected void initEnvironment(
- int configNodesNum, int dataNodesNum, int testWorkingRetryCount, boolean addAINode) {
- this.testWorkingRetryCount = testWorkingRetryCount;
+ int configNodesNum, int dataNodesNum, int retryCount, boolean addAINode) {
+ this.retryCount = retryCount;
this.configNodeWrapperList = new ArrayList<>();
this.dataNodeWrapperList = new ArrayList<>();
@@ -278,6 +278,7 @@ public abstract class AbstractEnv implements BaseEnv {
seedConfigNode,
testClassName,
testMethodName,
+ index,
EnvUtils.searchAvailablePorts(),
startTime);
aiNodeWrapperList.add(aiNodeWrapper);
@@ -400,7 +401,9 @@ public abstract class AbstractEnv implements BaseEnv {
@Override
public void cleanClusterEnvironment() {
List<AbstractNodeWrapper> allNodeWrappers =
- Stream.concat(this.dataNodeWrapperList.stream(), this.configNodeWrapperList.stream(), this.aiNodeWrapperList.strea())
+ Stream.concat(
+ dataNodeWrapperList.stream(),
+ Stream.concat(configNodeWrapperList.stream(), aiNodeWrapperList.stream()))
.collect(Collectors.toList());
allNodeWrappers.stream()
.findAny()
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
similarity index 54%
rename from integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java
rename to integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
index 5d713cd0dbd..233347c2b06 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
@@ -19,11 +19,9 @@
package org.apache.iotdb.it.env.cluster.node;
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
import org.apache.iotdb.it.framework.IoTDBTestLogger;
-import org.apache.iotdb.itbase.env.BaseNodeWrapper;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.file.PathUtils;
import org.slf4j.Logger;
import java.io.File;
@@ -31,21 +29,16 @@ import java.io.IOException;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.*;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.ML_NODE_NAME;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME;
import static org.apache.iotdb.it.env.cluster.EnvUtils.getTimeForLogDirectory;
-public class AINodeWrapper implements BaseNodeWrapper {
+public class AINodeWrapper extends AbstractNodeWrapper {
private static final Logger logger = IoTDBTestLogger.logger;
private final String testClassName;
private final String testMethodName;
- private final String nodeAddress;
- private Process instance;
- private final int nodePort;
private final long startTime;
private final String seedConfigNode;
@@ -73,77 +66,30 @@ public class AINodeWrapper implements BaseNodeWrapper {
prepareProcess.waitFor();
}
- public MLNodeWrapper(
+ public AINodeWrapper(
String seedConfigNode,
String testClassName,
String testMethodName,
+ int clusterIndex,
int[] port,
long startTime) {
+ super(testClassName, testMethodName, port, clusterIndex, false, startTime);
this.seedConfigNode = seedConfigNode;
this.testClassName = testClassName;
this.testMethodName = testMethodName;
- this.nodeAddress = "127.0.0.1";
- this.nodePort = port[0];
this.startTime = startTime;
}
- @Override
- public void createNodeDir() {
- // Copy templateNodePath to nodePath
- String destPath = getNodePath();
- try {
- try {
- if (new File(destPath).exists()) {
- PathUtils.deleteDirectory(Paths.get(destPath));
- }
- } catch (NoSuchFileException e) {
- // ignored
- }
- // Here we need to copy without follow symbolic links, so we can't use FileUtils directly.
- try (Stream<Path> s = Files.walk(Paths.get(TEMPLATE_NODE_PATH))) {
- s.forEach(
- source -> {
- Path destination =
- Paths.get(destPath, source.toString().substring(TEMPLATE_NODE_PATH.length()));
- try {
- Files.copy(
- source,
- destination,
- LinkOption.NOFOLLOW_LINKS,
- StandardCopyOption.COPY_ATTRIBUTES);
- } catch (IOException e) {
- logger.error("Got error copying files to node dest dir", e);
- throw new RuntimeException(e);
- }
- });
- }
- } catch (IOException ex) {
- logger.error("Copy node dir failed", ex);
- throw new AssertionError();
- }
- }
-
- @Override
- public void createLogDir() {
- try {
- // Make sure the log dir exist, as the first file is output by starting script directly.
- FileUtils.createParentDirectories(new File(getLogPath()));
- } catch (IOException ex) {
- logger.error("Copy node dir failed", ex);
- throw new AssertionError();
- }
- }
-
private String getLogPath() {
return getLogDirPath() + File.separator + getId() + ".log";
}
- private String getLogDirPath() {
+ public String getLogDirPath() {
return System.getProperty(USER_DIR)
+ File.separator
+ TARGET
+ File.separator
- + "mlnode-logs"
+ + "ainode-logs"
+ File.separator
+ getTestLogDirName()
+ File.separator
@@ -157,27 +103,6 @@ public class AINodeWrapper implements BaseNodeWrapper {
return testClassName + "_" + testMethodName;
}
- @Override
- public void destroyDir() {
- Exception lastException = null;
- for (int i = 0; i < 10; i++) {
- try {
- PathUtils.deleteDirectory(Paths.get(getNodePath()));
- return;
- } catch (IOException ex) {
- lastException = ex;
- try {
- TimeUnit.SECONDS.sleep(1);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new AssertionError("Delete node dir failed. " + e);
- }
- }
- }
- logger.error(lastException.getMessage());
- throw new AssertionError("Delete node dir failed.");
- }
-
@Override
public void start() {
try {
@@ -187,18 +112,18 @@ public class AINodeWrapper implements BaseNodeWrapper {
+ File.separator
+ TARGET
+ File.separator
- + ML_NODE_NAME
+ + AI_NODE_NAME
+ getPort();
String propertiesFile =
filePrefix + File.separator + CONFIG_PATH + File.separator + PROPERTIES_FILE;
// set attribute
replaceAttribute(
- "mln_target_config_node_list", this.seedConfigNode, propertiesFile, stdoutFile);
+ "ain_target_config_node_list", this.seedConfigNode, propertiesFile, stdoutFile);
replaceAttribute(
- "mln_inference_rpc_port", Integer.toString(getPort()), propertiesFile, stdoutFile);
+ "ain_inference_rpc_port", Integer.toString(getPort()), propertiesFile, stdoutFile);
- // start MLNode
+ // start AINode
List<String> start_command = new ArrayList<>();
start_command.add(SHELL_COMMAND);
start_command.add(filePrefix + File.separator + SCRIPT_PATH + File.separator + SCRIPT_FILE);
@@ -213,58 +138,55 @@ public class AINodeWrapper implements BaseNodeWrapper {
this.instance = processBuilder.start();
logger.info("In test {} {} started.", getTestLogDirName(), getId());
} catch (IOException | InterruptedException e) {
- throw new AssertionError("Start ML Node failed. " + e + Paths.get(""));
+ throw new AssertionError("Start AI Node failed. " + e + Paths.get(""));
}
}
@Override
- public void stop() {
- if (this.instance == null) {
- return;
- }
- this.instance.destroy();
- try {
- if (!this.instance.waitFor(20, TimeUnit.SECONDS)) {
- this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS);
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- logger.error("Waiting ML Node to shutdown error. %s", e);
- }
+ public int getMetricPort() {
+ // no metric currently
+ return -1;
}
@Override
- public String getIp() {
- return this.nodeAddress;
+ public String getId() {
+ return AI_NODE_NAME + getPort();
}
- @Override
- public int getPort() {
- return this.nodePort;
- }
+ /* Abstract methods, which must be implemented in ConfigNode and DataNode. */
+ public void reloadMutableFields() {}
+ ;
- @Override
- public int getMetricPort() {
- // no metric currently
- return -1;
+ public void renameFile() {}
+ ;
+
+ public String getSystemConfigPath() {
+ return "";
}
+ ;
- @Override
- public String getId() {
- return ML_NODE_NAME + getPort();
+ /** Return the node config file path specified through system variable */
+ public String getDefaultNodeConfigPath() {
+ return "";
}
+ ;
- @Override
- public String getIpAndPortString() {
- return this.getIp() + ":" + this.getPort();
+ /** Return the common config file path specified through system variable */
+ public String getDefaultCommonConfigPath() {
+ return "";
}
+ ;
- @Override
- public void dumpJVMSnapshot(String testCaseName) {
- // there is no JVM to dump for MLNode
+ public void addStartCmdParams(List<String> params) {}
+ ;
+
+ public String getSystemPropertiesPath() {
+ return "";
}
+ ;
- private String getNodePath() {
- return System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + getId();
+ public MppJVMConfig initVMConfig() {
+ throw new UnsupportedOperationException("AINode doesn't have JVM Config");
}
+ ;
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index 942b6a9e012..e6a26bd08db 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -125,11 +125,11 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
protected final MppJVMConfig jvmConfig;
protected final int clusterIndex;
protected final boolean isMultiCluster;
- private Process instance;
+ protected Process instance;
private final String nodeAddress;
private int nodePort;
- private int metricPort;
- private long startTime;
+ private final int metricPort;
+ private final long startTime;
private List<String> killPoints = new ArrayList<>();
/**
@@ -523,7 +523,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper {
}
@Override
- public final int getMetricPort() {
+ public int getMetricPort() {
return this.metricPort;
}
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MLClusterIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java
similarity index 100%
rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MLClusterIT.java
rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java
diff --git a/integration-test/src/test/java/org/apache/iotdb/mlnode/it/MLEnvIT.java b/integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java
similarity index 100%
rename from integration-test/src/test/java/org/apache/iotdb/mlnode/it/MLEnvIT.java
rename to integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java