You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by ba...@apache.org on 2024/04/25 03:55:56 UTC
(inlong) 02/02: [INLONG-10059][Agent] Fix the exception of installation package comparison (#10062)
This is an automated email from the ASF dual-hosted git repository.
baomingyu pushed a commit to branch branch-1.12
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit b26b20ad50e190f6b84fae235d2fd15aa6ec636b
Author: justinwwhuang <hw...@163.com>
AuthorDate: Thu Apr 25 09:49:09 2024 +0800
[INLONG-10059][Agent] Fix the exception of installation package comparison (#10062)
(cherry picked from commit 5e836457ccc1f26b2e20ee26640ffd5950ac001d)
---
.../inlong/agent/installer/ModuleManager.java | 112 +++++++++++++++++++--
.../src/test/java/installer/TestModuleManager.java | 8 +-
.../common/pojo/agent/installer/ConfigResult.java | 5 +-
.../service/core/impl/AgentServiceImpl.java | 3 +-
4 files changed, 107 insertions(+), 21 deletions(-)
diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
index dd438ed02a..db0dda7974 100755
--- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
+++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
@@ -28,6 +28,7 @@ import org.apache.inlong.agent.utils.ThreadUtils;
import org.apache.inlong.common.pojo.agent.installer.ConfigResult;
import org.apache.inlong.common.pojo.agent.installer.ModuleConfig;
import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum;
+import org.apache.inlong.common.pojo.agent.installer.PackageConfig;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -54,6 +55,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
@@ -78,6 +80,7 @@ public class ModuleManager extends AbstractDaemon {
public static final int DOWNLOAD_PACKAGE_READ_BUFF_SIZE = 1024 * 1024;
public static final String LOCAL_CONFIG_FILE = "modules.json";
private static final Logger LOGGER = LoggerFactory.getLogger(ModuleManager.class);
+ public static final int MAX_MODULE_SIZE = 10;
private final InstallerConfiguration conf;
private final String confPath;
private final BlockingQueue<ConfigResult> configQueue;
@@ -112,11 +115,8 @@ public class ModuleManager extends AbstractDaemon {
}
public void submitConfig(ConfigResult config) {
- if (config == null) {
- return;
- }
- if (config.getModuleList().isEmpty()) {
- LOGGER.error("module list should not be empty!");
+ if (!isConfigValid(config)) {
+ LOGGER.error("config is invalid !");
return;
}
configQueue.clear();
@@ -127,6 +127,94 @@ public class ModuleManager extends AbstractDaemon {
configQueue.add(config);
}
+ private boolean isConfigValid(ConfigResult config) {
+ if (config == null) {
+ LOGGER.error("config is null!");
+ return false;
+ }
+ if (config.getMd5() == null) {
+ LOGGER.error("modules md5 should not be null!");
+ return false;
+ }
+ if (config.getModuleList().isEmpty()) {
+ LOGGER.error("module list should not be empty!");
+ return false;
+ }
+ if (config.getModuleList().size() > MAX_MODULE_SIZE) {
+ LOGGER.error("module list {} over size {}!", config.getModuleList().size(), MAX_MODULE_SIZE);
+ return false;
+ }
+ for (int i = 0; i < config.getModuleList().size(); i++) {
+ if (!isModuleConfigValid(config.getModuleList().get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isModuleConfigValid(ModuleConfig module) {
+ if (module == null) {
+ LOGGER.error("module should not be null!");
+ return false;
+ }
+ if (module.getMd5() == null) {
+ LOGGER.error("module md5 should not be null!");
+ return false;
+ }
+ if (module.getName() == null) {
+ LOGGER.error("module name should not be null!");
+ return false;
+ }
+ if (module.getVersion() == null) {
+ LOGGER.error("module version should not be null!");
+ return false;
+ }
+ if (module.getInstallCommand() == null) {
+ LOGGER.error("module install cmd should not be null!");
+ return false;
+ }
+ if (module.getStartCommand() == null) {
+ LOGGER.error("module start cmd should not be null!");
+ return false;
+ }
+ if (module.getStopCommand() == null) {
+ LOGGER.error("module stop cmd should not be null!");
+ return false;
+ }
+ if (module.getCheckCommand() == null) {
+ LOGGER.error("module check cmd should not be null!");
+ return false;
+ }
+ if (!isPackageConfigValid(module.getPackageConfig())) {
+ return false;
+ }
+ return true;
+ }
+
+ private boolean isPackageConfigValid(PackageConfig packageConfig) {
+ if (packageConfig == null) {
+ LOGGER.error("module package config should not be null!");
+ return false;
+ }
+ if (packageConfig.getMd5() == null) {
+ LOGGER.error("package md5 should not be null!");
+ return false;
+ }
+ if (packageConfig.getFileName() == null) {
+ LOGGER.error("package file name should not be null!");
+ return false;
+ }
+ if (packageConfig.getDownloadUrl() == null) {
+ LOGGER.error("package url should not be null!");
+ return false;
+ }
+ if (packageConfig.getStoragePath() == null) {
+ LOGGER.error("package save path should not be null!");
+ return false;
+ }
+ return true;
+ }
+
public String getCurrentMd5() {
return currentMd5;
}
@@ -190,7 +278,7 @@ public class ModuleManager extends AbstractDaemon {
File jsonPath = new File(temp.getPath() + "/" + LOCAL_CONFIG_FILE);
try (BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(jsonPath), StandardCharsets.UTF_8))) {
- String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5).moduleNum(currentModules.size())
+ String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5)
.moduleList(currentModules.values().stream().collect(Collectors.toList())).build());
writer.write(curConfig);
writer.flush();
@@ -235,7 +323,8 @@ public class ModuleManager extends AbstractDaemon {
installModule(module);
saveModuleState(module.getId(), ModuleStateEnum.INSTALLED);
} else {
- LOGGER.info("check module {} package failed, change stated to new, will download package again",
+ LOGGER.info(
+ "check module {} package failed, change stated to new, will download package again",
module.getName());
saveModuleState(module.getId(), ModuleStateEnum.NEW);
}
@@ -425,7 +514,8 @@ public class ModuleManager extends AbstractDaemon {
}
private boolean downloadModule(ModuleConfig module) {
- LOGGER.info("download module {} begin with url {}", module.getId(), module.getPackageConfig().getDownloadUrl());
+ LOGGER.info("download module {} begin with url {}", module.getId(),
+ module.getPackageConfig().getDownloadUrl());
try {
URL url = new URL(module.getPackageConfig().getDownloadUrl());
URLConnection conn = url.openConnection();
@@ -433,7 +523,8 @@ public class ModuleManager extends AbstractDaemon {
authHeader.forEach((k, v) -> {
conn.setRequestProperty(k, v);
});
- String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
+ String path =
+ module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
try (InputStream inputStream = conn.getInputStream();
FileOutputStream outputStream = new FileOutputStream(path)) {
LOGGER.info("save path {}", path);
@@ -461,7 +552,7 @@ public class ModuleManager extends AbstractDaemon {
private boolean isPackageDownloaded(ModuleConfig module) {
String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
String fileMd5 = calcFileMd5(path);
- if (fileMd5.equals(module.getPackageConfig().getMd5())) {
+ if (Objects.equals(fileMd5, module.getPackageConfig().getMd5())) {
return true;
} else {
LOGGER.error("md5 not match! fileMd5 {} moduleMd5 {}", fileMd5, module.getPackageConfig().getMd5());
@@ -502,7 +593,6 @@ public class ModuleManager extends AbstractDaemon {
ret = new String(Hex.encodeHex(md.digest()));
} catch (NoSuchAlgorithmException e) {
LOGGER.error("calc file md5 NoSuchAlgorithmException", e);
-
} catch (IOException e) {
LOGGER.error("calc file md5 IOException", e);
}
diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java
index 719727852b..e5ccbd8cd3 100755
--- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java
+++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java
@@ -174,11 +174,11 @@ public class TestModuleManager {
configs.add(getModuleConfig(1, "inlong-agent", "inlong-agent-md5-185454", "1.0", 1,
"cd ~/inlong-agent/bin;sh agent.sh start", "cd ~/inlong-agent/bin;sh agent.sh stop",
"ps aux | grep core.AgentMain | grep java | grep -v grep | awk '{print $2}'",
- "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start",
- "echo empty uninstall cmd", "agent-release-1.12.0-SNAPSHOT-bin.tar.gz",
- "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.12.0-SNAPSHOT-bin.tar.gz",
+ "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.13.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start",
+ "echo empty uninstall cmd", "agent-release-1.13.0-SNAPSHOT-bin.tar.gz",
+ "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.13.0-SNAPSHOT-bin.tar.gz",
NEW_MD5));
- return ConfigResult.builder().moduleList(configs).moduleNum(1).md5("config-result-md5-193603").build();
+ return ConfigResult.builder().moduleList(configs).md5("config-result-md5-193603").build();
}
private ModuleConfig getModuleConfig(int id, String name, String md5, String version, Integer procNum,
diff --git a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java
index fcbbd0f5c1..14e504d211 100644
--- a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java
+++ b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java
@@ -42,10 +42,7 @@ public class ConfigResult {
* The md5 of the config result
*/
private String md5;
- /**
- * Number of module
- */
- private Integer moduleNum;
+
/**
* The list of module config list
*/
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
index 9eba178179..0972966246 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
@@ -844,8 +844,7 @@ public class AgentServiceImpl implements AgentService {
String jsonStr = GSON.toJson(configs);
String configMd5 = DigestUtils.md5Hex(jsonStr);
- ConfigResult configResult = ConfigResult.builder().moduleList(configs).moduleNum(configs.size())
- .md5(configMd5)
+ ConfigResult configResult = ConfigResult.builder().moduleList(configs).md5(configMd5)
.code(InstallerCode.SUCCESS)
.build();
LOGGER.info("success load module config, size = {}", configResult.getModuleList().size());