You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/09 13:56:47 UTC
[incubator-iotdb] branch rel/0.9 updated: Fix some HDFS config
issues in rel/0.9 (#1014)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.9
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/rel/0.9 by this push:
new 197ed91 Fix some HDFS config issues in rel/0.9 (#1014)
197ed91 is described below
commit 197ed914484fbcfea7eef20a52ca1db954848ce3
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Thu Apr 9 21:56:38 2020 +0800
Fix some HDFS config issues in rel/0.9 (#1014)
---
.../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 6 ++++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 29 ++++++++++++++--------
3 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
index df0189b..49765d4 100644
--- a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
+++ b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.hadoop.fileSystem;
-
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -27,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +36,10 @@ class HDFSConfUtil {
private static final Logger logger = LoggerFactory.getLogger(HDFSConfUtil.class);
static Configuration setConf(Configuration conf) {
+ if (!tsFileConfig.getTSFileStorageFs().equals(FSType.HDFS)) {
+ return conf;
+ }
+
try {
conf.addResource(new File(tsFileConfig.getCoreSitePath()).toURI().toURL());
conf.addResource(new File(tsFileConfig.getHdfsSitePath()).toURI().toURL());
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d13387e..4154566 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1187,6 +1187,10 @@ public class IoTDBConfig {
return hdfsIp.split(",");
}
+ String getRawHDFSIp() {
+ return hdfsIp;
+ }
+
public void setHdfsIp(String[] hdfsIp) {
this.hdfsIp = String.join(",", hdfsIp);
}
@@ -1219,6 +1223,10 @@ public class IoTDBConfig {
return dfsHaNamenodes.split(",");
}
+ String getRawDfsHaNamenodes() {
+ return dfsHaNamenodes;
+ }
+
public void setDfsHaNamenodes(String[] dfsHaNamenodes) {
this.dfsHaNamenodes = String.join(",", dfsHaNamenodes);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 13cc3ec..90a38f4 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -313,11 +313,12 @@ public class IoTDBDescriptor {
properties.getProperty("core_site_path", conf.getCoreSitePath()));
conf.setHdfsSitePath(
properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
- conf.setHdfsIp(properties.getProperty("hdfs_ip").split(","));
+ conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
conf.setDfsNameServices(
properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
- conf.setDfsHaNamenodes(properties.getProperty("dfs_ha_namenodes").split(","));
+ conf.setDfsHaNamenodes(
+ properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
conf.setDfsHaAutomaticFailoverEnabled(
Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled",
String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
@@ -336,22 +337,28 @@ public class IoTDBDescriptor {
// At the same time, set TSFileConfig
TSFileDescriptor.getInstance().getConfig()
- .setTSFileStorageFs(properties.getProperty("tsfile_storage_fs"));
- TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath(
+ .setTSFileStorageFs(
+ properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name()));
+ TSFileDescriptor.getInstance().getConfig().setCoreSitePath(
properties.getProperty("core_site_path", conf.getCoreSitePath()));
- TSFileDescriptor.getInstance().getConfig().setKerberosPrincipal(
+ TSFileDescriptor.getInstance().getConfig().setHdfsSitePath(
properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
TSFileDescriptor.getInstance().getConfig()
- .setHdfsIp(properties.getProperty("hdfs_ip").split(","));
- TSFileDescriptor.getInstance().getConfig().setHdfsPort(properties.getProperty("hdfs_port"));
+ .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
TSFileDescriptor.getInstance().getConfig()
- .setDfsNameServices(properties.getProperty("dfs_nameservices"));
+ .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
TSFileDescriptor.getInstance().getConfig()
- .setDfsHaNamenodes(properties.getProperty("dfs_ha_namenodes").split(","));
+ .setDfsNameServices(
+ properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
+ TSFileDescriptor.getInstance().getConfig()
+ .setDfsHaNamenodes(
+ properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
TSFileDescriptor.getInstance().getConfig().setDfsHaAutomaticFailoverEnabled(
- Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled")));
+ Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled",
+ String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
TSFileDescriptor.getInstance().getConfig().setDfsClientFailoverProxyProvider(
- properties.getProperty("dfs_client_failover_proxy_provider"));
+ properties.getProperty("dfs_client_failover_proxy_provider",
+ conf.getDfsClientFailoverProxyProvider()));
TSFileDescriptor.getInstance().getConfig().setUseKerberos(Boolean.parseBoolean(
properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath(