You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/11/07 14:23:37 UTC
[incubator-doris] branch master updated: Support hdfs broker
reading config from hdfs-site.xml (#2148)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2581d6d Support hdfs broker reading config from hdfs-site.xml (#2148)
2581d6d is described below
commit 2581d6d5746481d79074ca8a658c5a41dbfb380e
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Thu Nov 7 22:23:31 2019 +0800
Support hdfs broker reading config from hdfs-site.xml (#2148)
In the production environment, hdfs broker should read config from hdfs-site.xml instead of manual input due to the complexity of hdfs config
add dfs.namenode.kerberos.principal.pattern config for kerberos auth
fix some getting config code for brief
---
fs_brokers/apache_hdfs_broker/bin/start_broker.sh | 2 +-
fs_brokers/apache_hdfs_broker/build.sh | 1 +
fs_brokers/apache_hdfs_broker/conf/hdfs-site.xml | 20 +++++++++++++
.../doris/broker/hdfs/FileSystemManager.java | 35 +++++++++++-----------
4 files changed, 40 insertions(+), 18 deletions(-)
diff --git a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh
index a08140d..d0263d2 100755
--- a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh
+++ b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh
@@ -54,7 +54,7 @@ JAVA=$JAVA_HOME/bin/java
for f in $BROKER_HOME/lib/*.jar; do
CLASSPATH=$f:${CLASSPATH};
done
-export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib
+export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib:$BROKER_HOME/conf
while read line; do
envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`
diff --git a/fs_brokers/apache_hdfs_broker/build.sh b/fs_brokers/apache_hdfs_broker/build.sh
index 83753c7..e7922b6 100755
--- a/fs_brokers/apache_hdfs_broker/build.sh
+++ b/fs_brokers/apache_hdfs_broker/build.sh
@@ -44,6 +44,7 @@ install -d ${BROKER_OUTPUT}/bin ${BROKER_OUTPUT}/conf \
cp -r -p ${BROKER_HOME}/bin/*.sh ${BROKER_OUTPUT}/bin/
cp -r -p ${BROKER_HOME}/conf/*.conf ${BROKER_OUTPUT}/conf/
+cp -r -p ${BROKER_HOME}/conf/*.xml ${BROKER_OUTPUT}/conf/
cp -r -p ${BROKER_HOME}/conf/log4j.properties ${BROKER_OUTPUT}/conf/
cp -r -p ${BROKER_HOME}/target/lib/* ${BROKER_OUTPUT}/lib/
cp -r -p ${BROKER_HOME}/target/apache_hdfs_broker.jar ${BROKER_OUTPUT}/lib/
diff --git a/fs_brokers/apache_hdfs_broker/conf/hdfs-site.xml b/fs_brokers/apache_hdfs_broker/conf/hdfs-site.xml
new file mode 100644
index 0000000..d2ddf89
--- /dev/null
+++ b/fs_brokers/apache_hdfs_broker/conf/hdfs-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+</configuration>
diff --git a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index 8070e78..5e0d90f 100644
--- a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++ b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
@@ -69,7 +70,8 @@ public class FileSystemManager {
private static final String KERBEROS_PRINCIPAL = "kerberos_principal";
private static final String KERBEROS_KEYTAB = "kerberos_keytab";
private static final String KERBEROS_KEYTAB_CONTENT = "kerberos_keytab_content";
-
+ private static final String DFS_HA_NAMENODE_KERBEROS_PRINCIPAL_PATTERN =
+ "dfs.namenode.kerberos.principal.pattern";
// arguments for ha hdfs
private static final String DFS_NAMESERVICES_KEY = "dfs.nameservices";
private static final String DFS_HA_NAMENODES_PREFIX = "dfs.ha.namenodes.";
@@ -140,22 +142,17 @@ public class FileSystemManager {
"invalid hdfs path. authority is null");
}
}
- String username = properties.containsKey(USER_NAME_KEY) ? properties.get(USER_NAME_KEY) : "";
- String password = properties.containsKey(PASSWORD_KEY) ? properties.get(PASSWORD_KEY) : "";
- String dfsNameServices =
- properties.containsKey(DFS_NAMESERVICES_KEY) ? properties.get(DFS_NAMESERVICES_KEY) : "";
- String authentication = AUTHENTICATION_SIMPLE;
- if (properties.containsKey(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION)) {
- authentication = properties.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION);
- if (Strings.isNullOrEmpty(authentication)
- || (!authentication.equals(AUTHENTICATION_SIMPLE)
- && !authentication.equals(AUTHENTICATION_KERBEROS))) {
- logger.warn("invalid authentication:" + authentication);
- throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT,
- "invalid authentication:" + authentication);
- }
+ String username = properties.getOrDefault(USER_NAME_KEY, "");
+ String password = properties.getOrDefault(PASSWORD_KEY, "");
+ String dfsNameServices = properties.getOrDefault(DFS_NAMESERVICES_KEY, "");
+ String authentication = properties.getOrDefault(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ AUTHENTICATION_SIMPLE);
+ if (Strings.isNullOrEmpty(authentication) || (!authentication.equals(AUTHENTICATION_SIMPLE)
+ && !authentication.equals(AUTHENTICATION_KERBEROS))) {
+ logger.warn("invalid authentication:" + authentication);
+ throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT,
+ "invalid authentication:" + authentication);
}
-
String hdfsUgi = username + "," + password;
FileSystemIdentity fileSystemIdentity = null;
BrokerFileSystem fileSystem = null;
@@ -204,7 +201,7 @@ public class FileSystemManager {
if (fileSystem.getDFSFileSystem() == null) {
logger.info("could not find file system for path " + path + " create a new one");
// create a new filesystem
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
// TODO get this param from properties
// conf.set("dfs.replication", "2");
String tmpFilePath = null;
@@ -293,6 +290,10 @@ public class FileSystemManager {
if (properties.containsKey(FS_DEFAULTFS_KEY)) {
conf.set(FS_DEFAULTFS_KEY, properties.get(FS_DEFAULTFS_KEY));
}
+ if (properties.containsKey(DFS_HA_NAMENODE_KERBEROS_PRINCIPAL_PATTERN)) {
+ conf.set(DFS_HA_NAMENODE_KERBEROS_PRINCIPAL_PATTERN,
+ properties.get(DFS_HA_NAMENODE_KERBEROS_PRINCIPAL_PATTERN));
+ }
}
FileSystem dfsFileSystem = FileSystem.get(pathUri.getUri(), conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org