You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ji...@apache.org on 2022/04/01 04:29:40 UTC
[incubator-doris-manager] branch master updated: Fix the path problem of broker when hosting cluster (#28)
This is an automated email from the ASF dual-hosted git repository.
jiafengzheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris-manager.git
The following commit(s) were added to refs/heads/master by this push:
new 0071d6c Fix the path problem of broker when hosting cluster (#28)
0071d6c is described below
commit 0071d6cad5c3f41f80e2aae93c4a3d735ed095e5
Author: songchuanyuan <46...@users.noreply.github.com>
AuthorDate: Fri Apr 1 12:29:35 2022 +0800
Fix the path problem of broker when hosting cluster (#28)
Fix the path problem of broker when hosting cluster
---
.../service/heartbeat/DorisInstanceOperator.java | 94 ++++++++++++++++++----
.../common/util/ServerAndAgentConstant.java | 21 +++++
.../manager/DorisClusterInstanceManager.java | 2 +-
.../control/manager/DorisClusterModuleManager.java | 2 +-
4 files changed, 101 insertions(+), 18 deletions(-)
diff --git a/manager/dm-agent/src/main/java/org/apache/doris/manager/agent/service/heartbeat/DorisInstanceOperator.java b/manager/dm-agent/src/main/java/org/apache/doris/manager/agent/service/heartbeat/DorisInstanceOperator.java
index 961f4a4..621049e 100644
--- a/manager/dm-agent/src/main/java/org/apache/doris/manager/agent/service/heartbeat/DorisInstanceOperator.java
+++ b/manager/dm-agent/src/main/java/org/apache/doris/manager/agent/service/heartbeat/DorisInstanceOperator.java
@@ -38,6 +38,11 @@ import java.util.Map;
@Component
public class DorisInstanceOperator {
+ // Actual broker installation path
+ // For compatibility, the actual broker deployment folder name may be baidu_doris_broker
+ // or apache_hdfs_broker when the cluster is hosted
+ private String brokerInstallationPath = "";
+
// Download installation package
public boolean downloadInstancePackage(String moudleName, String installInfo, String packageInfo) {
File packageFile = Paths.get(installInfo, moudleName).toFile();
@@ -82,7 +87,15 @@ public class DorisInstanceOperator {
}
confFile = Paths.get(installInfo, moudleName, "conf", ServerAndAgentConstant.BE_CONF_FILE).toFile();
} else {
- confFile = Paths.get(installInfo, moudleName, "conf", ServerAndAgentConstant.BROKER_CONF_FILE).toFile();
+ String actualBrokerPath = getBrokerInstallationPath(installInfo);
+ String configFileName = ServerAndAgentConstant.BROKER_CONF_FILE;
+
+ if (actualBrokerPath.equals(ServerAndAgentConstant.BAIDU_BROKER_INIT_SUB_DIR)) {
+ log.debug("Baidu doris broker config");
+ configFileName = ServerAndAgentConstant.BAIDU_BROKER_CONF_FILE;
+ parms.putAll(ServerAndAgentConstant.BAIDU_BROKER_CONFIG_DEDAULT);
+ }
+ confFile = Paths.get(installInfo, actualBrokerPath, "conf", configFileName).toFile();
}
// Create a new profile
@@ -108,17 +121,22 @@ public class DorisInstanceOperator {
public boolean startInstance(String moudleName, String installInfo, String followerEndpoint) {
log.info("begin to start {} instance", moudleName);
try {
+ if (moudleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ moudleName = getBrokerInstallationPath(installInfo);
+ }
+
int mainProcPid = processIsRunning(moudleName, installInfo);
if (mainProcPid == -1) {
log.info("{} instance not running, start it", moudleName);
- String startScript = ServerAndAgentConstant.FE_START_SCRIPT;
- if (followerEndpoint != null && !followerEndpoint.isEmpty()) {
- startScript += " --helper " + followerEndpoint;
- }
-
- if (moudleName.equals(ServerAndAgentConstant.BE_NAME)) {
+ String startScript = "";
+ if (moudleName.equals(ServerAndAgentConstant.FE_NAME)) {
+ startScript = ServerAndAgentConstant.FE_START_SCRIPT;
+ if (followerEndpoint != null && !followerEndpoint.isEmpty()) {
+ startScript += " --helper " + followerEndpoint;
+ }
+ } else if (moudleName.equals(ServerAndAgentConstant.BE_NAME)) {
startScript = ServerAndAgentConstant.BE_START_SCRIPT;
- } else if (moudleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ } else {
startScript = ServerAndAgentConstant.BROKER_START_SCRIPT;
}
startScript += " --daemon";
@@ -138,13 +156,19 @@ public class DorisInstanceOperator {
public boolean stopInstance(String moudleName, String installInfo) {
log.info("begin to stop {} instance", moudleName);
try {
+ if (moudleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ moudleName = getBrokerInstallationPath(installInfo);
+ }
+
int mainProcPid = processIsRunning(moudleName, installInfo);
if (mainProcPid > -1) {
log.info("{} instance is running, stop it", moudleName);
- String stopScript = ServerAndAgentConstant.FE_STOP_SCRIPT;
- if (moudleName.equals(ServerAndAgentConstant.BE_NAME)) {
+ String stopScript = "";
+ if (moudleName.equals(ServerAndAgentConstant.FE_NAME)) {
+ stopScript = ServerAndAgentConstant.FE_STOP_SCRIPT;
+ } else if (moudleName.equals(ServerAndAgentConstant.BE_NAME)) {
stopScript = ServerAndAgentConstant.BE_STOP_SCRIPT;
- } else if (moudleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ } else {
stopScript = ServerAndAgentConstant.BROKER_STOP_SCRIPT;
}
executePkgShellScript(stopScript, installInfo, moudleName, Maps.newHashMap());
@@ -170,6 +194,9 @@ public class DorisInstanceOperator {
// Check whether the instance has been installed and started
public boolean checkInstanceDeploy(String moudleName, String installInfo) {
try {
+ if (moudleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ moudleName = getBrokerInstallationPath(installInfo);
+ }
int bePid = processIsRunning(moudleName, installInfo);
if (bePid < 0) {
return false;
@@ -192,14 +219,22 @@ public class DorisInstanceOperator {
*/
private int processIsRunning(String moduleName, String runningDir) throws Exception {
- String processName = ServerAndAgentConstant.FE_PID_NAME;
- String pidFileName = ServerAndAgentConstant.FE_PID_FILE;
- if (moduleName.equals(ServerAndAgentConstant.BE_NAME)) {
+ String processName = "";
+ String pidFileName = "";
+
+ if (moduleName.equals(ServerAndAgentConstant.FE_NAME)) {
+ processName = ServerAndAgentConstant.FE_PID_NAME;
+ pidFileName = ServerAndAgentConstant.FE_PID_FILE;
+ } else if (moduleName.equals(ServerAndAgentConstant.BE_NAME)) {
processName = ServerAndAgentConstant.BE_PID_NAME;
pidFileName = ServerAndAgentConstant.BE_PID_FILE;
- } else if (moduleName.equals(ServerAndAgentConstant.BROKER_NAME)) {
+ } else {
processName = ServerAndAgentConstant.BROKER_PID_NAME;
- pidFileName = ServerAndAgentConstant.BROKER_PID_FILE;
+ if (moduleName.equals(ServerAndAgentConstant.BAIDU_BROKER_INIT_SUB_DIR)) {
+ pidFileName = ServerAndAgentConstant.BAIDU_BROKER_PID_FILE;
+ } else {
+ pidFileName = ServerAndAgentConstant.BROKER_PID_FILE;
+ }
}
int pid = getPid(processName);
@@ -300,6 +335,7 @@ public class DorisInstanceOperator {
int index = scripts.indexOf(":") + 1;
scripts = scripts.substring(0, index) + "//" + scripts.substring(index + 1);
final String shellCmd = "sh " + scripts;
+
log.info("begin to execute: `" + shellCmd + "`");
executeShell(shellCmd, environment);
}
@@ -328,4 +364,30 @@ public class DorisInstanceOperator {
}
}
}
+
+ // Set the actual path of the broker
+ private String getBrokerInstallationPath(String installInfo) {
+ if (brokerInstallationPath.isEmpty()) {
+
+ File brokerFile = Paths.get(installInfo, ServerAndAgentConstant.BROKER_NAME).toFile();
+ if (brokerFile.exists()) {
+ return ServerAndAgentConstant.BROKER_NAME;
+ }
+
+ brokerFile = Paths.get(installInfo, ServerAndAgentConstant.BROKER_INIT_SUB_DIR).toFile();
+ if (brokerFile.exists()) {
+ return ServerAndAgentConstant.BROKER_INIT_SUB_DIR;
+ }
+
+ brokerFile = Paths.get(installInfo, ServerAndAgentConstant.BAIDU_BROKER_INIT_SUB_DIR).toFile();
+ if (brokerFile.exists()) {
+ return ServerAndAgentConstant.BAIDU_BROKER_INIT_SUB_DIR;
+ }
+
+ return ServerAndAgentConstant.BROKER_NAME;
+
+ } else {
+ return brokerInstallationPath;
+ }
+ }
}
diff --git a/manager/dm-common/src/main/java/org/apache/doris/manager/common/util/ServerAndAgentConstant.java b/manager/dm-common/src/main/java/org/apache/doris/manager/common/util/ServerAndAgentConstant.java
index f595cd1..dbc813f 100644
--- a/manager/dm-common/src/main/java/org/apache/doris/manager/common/util/ServerAndAgentConstant.java
+++ b/manager/dm-common/src/main/java/org/apache/doris/manager/common/util/ServerAndAgentConstant.java
@@ -17,6 +17,10 @@
package org.apache.doris.manager.common.util;
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
public class ServerAndAgentConstant {
private ServerAndAgentConstant() {
@@ -34,6 +38,7 @@ public class ServerAndAgentConstant {
// The path of borker module initialization when the installation package is downloaded
public static final String BROKER_INIT_SUB_DIR = "apache_hdfs_broker";
+ public static final String BAIDU_BROKER_INIT_SUB_DIR = "baidu_doris_broker";
public static final String DORIS_INSTALL_HOME_EVN = "DORIS_INSTALL_HOME";
public static final String DORIS_PACKAGE_URL_ENV = "DORIS_PACKAGE_URL";
@@ -41,6 +46,7 @@ public class ServerAndAgentConstant {
public static final String FE_PID_FILE = "fe.pid";
public static final String BE_PID_FILE = "be.pid";
public static final String BROKER_PID_FILE = "apache_hdfs_broker.pid";
+ public static final String BAIDU_BROKER_PID_FILE = "baidu_doris_broker.pid";
public static final String FE_PID_NAME = "PaloFe";
public static final String BE_PID_NAME = "palo_be";
@@ -57,6 +63,7 @@ public class ServerAndAgentConstant {
public static final String FE_CONF_FILE = "fe.conf";
public static final String BE_CONF_FILE = "be.conf";
public static final String BROKER_CONF_FILE = "apache_hdfs_broker.conf";
+ public static final String BAIDU_BROKER_CONF_FILE = "baidu_doris_broker.conf";
public static final String PACKAGE_DOWNLOAD_SCRIPT = "download_doris.sh";
@@ -68,4 +75,18 @@ public class ServerAndAgentConstant {
public static final String BE_HEARTBEAT_SERVICE = "be_heartbeat";
public static final String BROKER_PRC_SERVICE = "broker_rpc";
+ public static final Map<String, String> BAIDU_BROKER_CONFIG_DEDAULT;
+
+ static {
+ BAIDU_BROKER_CONFIG_DEDAULT = Maps.newHashMap();
+ BAIDU_BROKER_CONFIG_DEDAULT.put("afs_filesystem_impl", "org.apache.hadoop.fs.DFileSystem");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("hdfs_filesystem_impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("bos_filesystem_impl", "org.apache.hadoop.fs.bos.BaiduBosFileSystem");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("afs_agent_port", "20001");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("hdfs_agent_port", "20002");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("afs_client_auth_method", "3");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("hdfs_client_auth_method", "2");
+ BAIDU_BROKER_CONFIG_DEDAULT.put("bos_client_auth_method", "2");
+ }
+
}
diff --git a/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterInstanceManager.java b/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterInstanceManager.java
index 1452681..ae7b0d9 100644
--- a/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterInstanceManager.java
+++ b/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterInstanceManager.java
@@ -51,7 +51,7 @@ public class DorisClusterInstanceManager {
public long initOperation(long clusterId, ClusterModuleEntity moudle, long nodeId) {
// TODO:Judge whether node can deploy this instance
- log.info("create a new instance for cluster {} moudle {} on node {}", clusterId, moudle.getId(), nodeId);
+ log.info("create a new instance for cluster {} moudle {} on node {}", clusterId, moudle.getModuleName(), nodeId);
ResourceNodeEntity nodeEntity = nodeRepository.findById(nodeId).get();
ClusterInstanceEntity instanceEntity = new ClusterInstanceEntity(clusterId, moudle.getId(), nodeId,
diff --git a/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterModuleManager.java b/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterModuleManager.java
index b68fdeb..492e19d 100644
--- a/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterModuleManager.java
+++ b/manager/dm-server/src/main/java/org/apache/doris/stack/control/manager/DorisClusterModuleManager.java
@@ -59,7 +59,7 @@ public class DorisClusterModuleManager {
private DorisClusterInstanceManager instanceManager;
public long initOperation(long clusterId, DorisClusterModuleResourceConfig resourceConfig) {
- log.info("create module for cluster {}", clusterId);
+ log.info("create module {} for cluster {}", resourceConfig.getModuleName(), clusterId);
ClusterModuleEntity moduleEntity = new ClusterModuleEntity(clusterId, resourceConfig.getModuleName());
ClusterModuleEntity newModuleEntity = clusterModuleRepository.save(moduleEntity);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org