You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/11/19 02:02:35 UTC
[incubator-iotdb] branch rel/0.9 updated: [IOTDB-304] Fix bug of
incomplete HDFS URI (#551)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.9
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/rel/0.9 by this push:
new 31b144d [IOTDB-304] Fix bug of incomplete HDFS URI (#551)
31b144d is described below
commit 31b144dc17c70cd82a1b0541b1580c2da26887ea
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Nov 19 09:54:26 2019 +0800
[IOTDB-304] Fix bug of incomplete HDFS URI (#551)
* Add configuration core_site_path and hdfs_site_path
(cherry picked from commit fa27882c8b6414e268440ad7453c18fe672569f0)
---
.../UserGuide/3-Server/4-Config Manual.md | 18 +++++++++++
.../3-Writing Data on HDFS.md | 18 +++++++++++
.../UserGuide/3-Server/4-Config Manual.md | 18 +++++++++++
.../3-Writing Data on HDFS.md | 18 +++++++++++
.../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 37 ++++++++++++++++------
.../apache/iotdb/hadoop/fileSystem/HDFSFile.java | 3 +-
.../resources/conf/iotdb-engine.properties | 6 ++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +++++++++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 9 +++++-
.../engine/storagegroup/StorageGroupProcessor.java | 6 ++--
.../iotdb/tsfile/common/conf/TSFileConfig.java | 24 ++++++++++++++
11 files changed, 166 insertions(+), 18 deletions(-)
diff --git a/docs/Documentation-CHN/UserGuide/3-Server/4-Config Manual.md b/docs/Documentation-CHN/UserGuide/3-Server/4-Config Manual.md
index 6914218..f635bc8 100644
--- a/docs/Documentation-CHN/UserGuide/3-Server/4-Config Manual.md
+++ b/docs/Documentation-CHN/UserGuide/3-Server/4-Config Manual.md
@@ -302,6 +302,24 @@
|默认值|LOCAL |
|改后生效方式|重启服务器生效|
+* core\_site\_path
+
+|Name| core\_site\_path |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置core-site.xml的绝对路径|
+|类型| String |
+|默认值|/etc/hadoop/conf/core-site.xml |
+|改后生效方式|重启服务器生效|
+
+* hdfs\_site\_path
+
+|Name| hdfs\_site\_path |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置hdfs-site.xml的绝对路径|
+|类型| String |
+|默认值|/etc/hadoop/conf/hdfs-site.xml |
+|改后生效方式|重启服务器生效|
+
* hdfs\_ip
|名字| hdfs\_ip |
diff --git a/docs/Documentation-CHN/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md b/docs/Documentation-CHN/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
index 663028a..7cdd806 100644
--- a/docs/Documentation-CHN/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
+++ b/docs/Documentation-CHN/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
@@ -52,6 +52,24 @@
|默认值|LOCAL |
|改后生效方式|重启服务器生效|
+* core\_site\_path
+
+|Name| core\_site\_path |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置core-site.xml的绝对路径|
+|类型| String |
+|默认值|/etc/hadoop/conf/core-site.xml |
+|改后生效方式|重启服务器生效|
+
+* hdfs\_site\_path
+
+|Name| hdfs\_site\_path |
+|:---:|:---|
+|描述| 在Tsfile和相关数据文件存储到HDFS的情况下用于配置hdfs-site.xml的绝对路径|
+|类型| String |
+|默认值|/etc/hadoop/conf/hdfs-site.xml |
+|改后生效方式|重启服务器生效|
+
* hdfs\_ip
|名字| hdfs\_ip |
diff --git a/docs/Documentation/UserGuide/3-Server/4-Config Manual.md b/docs/Documentation/UserGuide/3-Server/4-Config Manual.md
index 450d645..061197a 100644
--- a/docs/Documentation/UserGuide/3-Server/4-Config Manual.md
+++ b/docs/Documentation/UserGuide/3-Server/4-Config Manual.md
@@ -337,6 +337,24 @@ The detail of each variables are as follows:
|Default|LOCAL |
|Effective|After restart system|
+* core\_site\_path
+
+|Name| core\_site\_path |
+|:---:|:---|
+|Description| Absolute file path of core-site.xml if Tsfile and related data files are stored in HDFS.|
+|Type| String |
+|Default|/etc/hadoop/conf/core-site.xml |
+|Effective|After restart system|
+
+* hdfs\_site\_path
+
+|Name| hdfs\_site\_path |
+|:---:|:---|
+|Description| Absolute file path of hdfs-site.xml if Tsfile and related data files are stored in HDFS.|
+|Type| String |
+|Default|/etc/hadoop/conf/hdfs-site.xml |
+|Effective|After restart system|
+
* hdfs\_ip
|Name| hdfs\_ip |
diff --git a/docs/Documentation/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md b/docs/Documentation/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
index 1cc8ce6..27571f8 100644
--- a/docs/Documentation/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
+++ b/docs/Documentation/UserGuide/8-System Design (Developer)/3-Writing Data on HDFS.md
@@ -52,6 +52,24 @@ Edit user config in `iotdb-engine.properties`. Related configurations are:
|Default|LOCAL |
|Effective|After restart system|
+* core\_site\_path
+
+|Name| core\_site\_path |
+|:---:|:---|
+|Description| Absolute file path of core-site.xml if Tsfile and related data files are stored in HDFS.|
+|Type| String |
+|Default|/etc/hadoop/conf/core-site.xml |
+|Effective|After restart system|
+
+* hdfs\_site\_path
+
+|Name| hdfs\_site\_path |
+|:---:|:---|
+|Description| Absolute file path of hdfs-site.xml if Tsfile and related data files are stored in HDFS.|
+|Type| String |
+|Default|/etc/hadoop/conf/hdfs-site.xml |
+|Effective|After restart system|
+
* hdfs\_ip
|Name| hdfs\_ip |
diff --git a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
index 3cb85ed..df0189b 100644
--- a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
+++ b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
@@ -20,15 +20,30 @@
package org.apache.iotdb.hadoop.fileSystem;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class HDFSConfUtil {
+class HDFSConfUtil {
private static TSFileConfig tsFileConfig = TSFileDescriptor.getInstance().getConfig();
+ private static final Logger logger = LoggerFactory.getLogger(HDFSConfUtil.class);
+
+ static Configuration setConf(Configuration conf) {
+ try {
+ conf.addResource(new File(tsFileConfig.getCoreSitePath()).toURI().toURL());
+ conf.addResource(new File(tsFileConfig.getHdfsSitePath()).toURI().toURL());
+ } catch (MalformedURLException e) {
+ logger.error("Failed to add resource core-site.xml {} and hdfs-site.xml {}. ",
+ tsFileConfig.getCoreSitePath(), tsFileConfig.getHdfsSitePath(), e);
+ }
- public static Configuration setConf(Configuration conf) {
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
@@ -53,19 +68,21 @@ public class HDFSConfUtil {
}
// Kerberos configuration
- if(tsFileConfig.isUseKerberos()){
+ if (tsFileConfig.isUseKerberos()) {
conf.set("hadoop.security.authorization", "true");
conf.set("hadoop.security.authentication", "kerberos");
conf.set("dfs.block.access.token.enable", "true");
- conf.set("dfs.namenode.kerberos.principal", tsFileConfig.getKerberosPrincipal());
- conf.set("dfs.namenode.keytab.file", tsFileConfig.getKerberosKeytabFilePath());
- conf.set("dfs.secondary.namenode.kerberos.principal", tsFileConfig.getKerberosPrincipal());
- conf.set("dfs.secondary.namenode.keytab.file", tsFileConfig.getKerberosKeytabFilePath());
- conf.set("dfs.datanode.kerberos.principal", tsFileConfig.getKerberosPrincipal());
- conf.set("dfs.datanode.keytab.file", tsFileConfig.getKerberosKeytabFilePath());
+ UserGroupInformation.setConfiguration(conf);
+ try {
+ UserGroupInformation.loginUserFromKeytab(tsFileConfig.getKerberosPrincipal(),
+ tsFileConfig.getKerberosKeytabFilePath());
+ } catch (IOException e) {
+ logger.error("Failed to login user from key tab. User: {}, path:{}. ",
+ tsFileConfig.getKerberosPrincipal(), tsFileConfig.getKerberosKeytabFilePath(), e);
+ }
}
return conf;
}
-}
+}
\ No newline at end of file
diff --git a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSFile.java b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSFile.java
index fd12da1..3b23bfe 100644
--- a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSFile.java
+++ b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSFile.java
@@ -39,13 +39,12 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HDFSFile extends File {
+ private static final long serialVersionUID = -8419827359081949547L;
private Path hdfsPath;
private FileSystem fs;
private static final Logger logger = LoggerFactory.getLogger(HDFSFile.class);
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index e31de76..9ef203a 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -131,6 +131,12 @@ timestamp_precision=ms
# TSFile storage file system. Currently, Tsfile are supported to be stored in LOCAL file system or HDFS.
tsfile_storage_fs=LOCAL
+# If using HDFS, the absolute file path of Hadoop core-site.xml should be configured
+core_site_path=/etc/hadoop/conf/core-site.xml
+
+# If using HDFS, the absolute file path of Hadoop hdfs-site.xml should be configured
+hdfs_site_path=/etc/hadoop/conf/hdfs-site.xml
+
# If using HDFS, hadoop ip can be configured. If there are more than one hdfs_ip, Hadoop HA is used
hdfs_ip=localhost
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 7f6ea30..d13387e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -397,6 +397,16 @@ public class IoTDBConfig {
private FSType tsFileStorageFs = FSType.LOCAL;
/**
+ * Default core-site.xml file path is /etc/hadoop/conf/core-site.xml
+ */
+ private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
+
+ /**
+ * Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml
+ */
+ private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml";
+
+ /**
* Default HDFS ip is localhost
*/
private String hdfsIp = "localhost";
@@ -426,7 +436,6 @@ public class IoTDBConfig {
*/
private String dfsClientFailoverProxyProvider = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
-
/**
* whether use kerberos to authenticate hdfs
*/
@@ -1158,6 +1167,22 @@ public class IoTDBConfig {
this.tsFileStorageFs = FSType.valueOf(tsFileStorageFs);
}
+ public String getCoreSitePath() {
+ return coreSitePath;
+ }
+
+ public void setCoreSitePath(String coreSitePath) {
+ this.coreSitePath = coreSitePath;
+ }
+
+ public String getHdfsSitePath() {
+ return hdfsSitePath;
+ }
+
+ public void setHdfsSitePath(String hdfsSitePath) {
+ this.hdfsSitePath = hdfsSitePath;
+ }
+
public String[] getHdfsIp() {
return hdfsIp.split(",");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 8c4661e..13cc3ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -309,6 +309,10 @@ public class IoTDBDescriptor {
conf.setTsFileStorageFs(properties.getProperty("tsfile_storage_fs",
conf.getTsFileStorageFs().toString()));
+ conf.setCoreSitePath(
+ properties.getProperty("core_site_path", conf.getCoreSitePath()));
+ conf.setHdfsSitePath(
+ properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
conf.setHdfsIp(properties.getProperty("hdfs_ip").split(","));
conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
conf.setDfsNameServices(
@@ -333,6 +337,10 @@ public class IoTDBDescriptor {
// At the same time, set TSFileConfig
TSFileDescriptor.getInstance().getConfig()
.setTSFileStorageFs(properties.getProperty("tsfile_storage_fs"));
+ TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath(
+ properties.getProperty("core_site_path", conf.getCoreSitePath()));
+ TSFileDescriptor.getInstance().getConfig().setKerberosPrincipal(
+ properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
TSFileDescriptor.getInstance().getConfig()
.setHdfsIp(properties.getProperty("hdfs_ip").split(","));
TSFileDescriptor.getInstance().getConfig().setHdfsPort(properties.getProperty("hdfs_port"));
@@ -351,7 +359,6 @@ public class IoTDBDescriptor {
TSFileDescriptor.getInstance().getConfig().setKerberosPrincipal(
properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
-
// set tsfile-format config
TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(Integer
.parseInt(properties.getProperty("group_size_in_byte",
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 990ae28..fc64ddc 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -24,7 +24,6 @@ import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFF
import java.io.File;
import java.io.IOException;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -524,10 +523,9 @@ public class StorageGroupProcessor {
baseDir = DirectoryManager.getInstance().getNextFolderForUnSequenceFile();
}
fsFactory.getFile(baseDir, storageGroupName).mkdirs();
-
- String filePath = Paths.get(baseDir, storageGroupName,
+ String filePath = baseDir + File.separator + storageGroupName + File.separator +
System.currentTimeMillis() + IoTDBConstant.TSFILE_NAME_SEPARATOR + versionController
- .nextVersion()).toString() + IoTDBConstant.TSFILE_NAME_SEPARATOR + "0" + TSFILE_SUFFIX;
+ .nextVersion() + IoTDBConstant.TSFILE_NAME_SEPARATOR + "0" + TSFILE_SUFFIX;
if (sequence) {
return new TsFileProcessor(storageGroupName, fsFactory.getFile(filePath),
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 079378b..8ff6d21 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -144,6 +144,14 @@ public class TSFileConfig {
*/
private FSType TSFileStorageFs = FSType.LOCAL;
/**
+ * Default core-site.xml file path is /etc/hadoop/conf/core-site.xml
+ */
+ private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
+ /**
+ * Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml
+ */
+ private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml";
+ /**
* Default hdfs ip is localhost
*/
private String hdfsIp = "localhost";
@@ -373,6 +381,22 @@ public class TSFileConfig {
this.TSFileStorageFs = FSType.valueOf(TSFileStorageFs);
}
+ public String getCoreSitePath() {
+ return coreSitePath;
+ }
+
+ public void setCoreSitePath(String coreSitePath) {
+ this.coreSitePath = coreSitePath;
+ }
+
+ public String getHdfsSitePath() {
+ return hdfsSitePath;
+ }
+
+ public void setHdfsSitePath(String hdfsSitePath) {
+ this.hdfsSitePath = hdfsSitePath;
+ }
+
public String[] getHdfsIp() {
return hdfsIp.split(",");
}