You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by es...@apache.org on 2017/02/03 09:00:39 UTC
[37/50] [abbrv] incubator-hawq git commit: HAWQ-1193. Add
createEncryption, getEZForPath, listEncryptionZones RPC for libhdfs3.
HAWQ-1193. Add createEncryption, getEZForPath, listEncryptionZones RPC for libhdfs3.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/517e6d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/517e6d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/517e6d26
Branch: refs/heads/2.1.0.0-incubating
Commit: 517e6d26cadff3a0fc03af1f36e8302bf454e573
Parents: efa1230
Author: ivan <iw...@pivotal.io>
Authored: Tue Dec 20 16:51:18 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Thu Jan 19 16:10:32 2017 +0800
----------------------------------------------------------------------
depends/libhdfs3/mock/MockFileSystemInter.h | 5 +
.../libhdfs3/src/client/EncryptionZoneInfo.h | 80 +++++++++++++
.../src/client/EncryptionZoneIterator.cpp | 86 ++++++++++++++
.../src/client/EncryptionZoneIterator.h | 56 +++++++++
.../libhdfs3/src/client/FileEncryptionInfo.h | 93 +++++++++++++++
depends/libhdfs3/src/client/FileStatus.h | 19 +++-
depends/libhdfs3/src/client/FileSystem.cpp | 54 +++++++++
depends/libhdfs3/src/client/FileSystem.h | 32 ++++++
depends/libhdfs3/src/client/FileSystemImpl.cpp | 88 ++++++++++++++
depends/libhdfs3/src/client/FileSystemImpl.h | 40 +++++++
depends/libhdfs3/src/client/FileSystemInter.h | 40 +++++++
depends/libhdfs3/src/client/Hdfs.cpp | 114 ++++++++++++++++++-
depends/libhdfs3/src/client/hdfs.h | 63 ++++++++++
.../src/proto/ClientNamenodeProtocol.proto | 8 ++
depends/libhdfs3/src/proto/datatransfer.proto | 1 +
depends/libhdfs3/src/proto/encryption.proto | 67 +++++++++++
depends/libhdfs3/src/proto/hdfs.proto | 60 ++++++++++
depends/libhdfs3/src/rpc/RpcAuth.h | 2 +-
depends/libhdfs3/src/rpc/RpcChannel.cpp | 4 +-
depends/libhdfs3/src/server/Namenode.h | 35 +++++-
depends/libhdfs3/src/server/NamenodeImpl.cpp | 73 ++++++++++++
depends/libhdfs3/src/server/NamenodeImpl.h | 9 ++
depends/libhdfs3/src/server/NamenodeProxy.cpp | 24 ++++
depends/libhdfs3/src/server/NamenodeProxy.h | 7 ++
depends/libhdfs3/src/server/RpcHelper.h | 31 +++++
.../libhdfs3/test/function/TestCInterface.cpp | 42 +++++++
.../libhdfs3/test/function/TestFileSystem.cpp | 33 ++++++
.../libhdfs3/test/function/TestOutputStream.cpp | 12 ++
28 files changed, 1171 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/mock/MockFileSystemInter.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/mock/MockFileSystemInter.h b/depends/libhdfs3/mock/MockFileSystemInter.h
index 4d91c73..336db7e 100644
--- a/depends/libhdfs3/mock/MockFileSystemInter.h
+++ b/depends/libhdfs3/mock/MockFileSystemInter.h
@@ -101,6 +101,11 @@ public:
MOCK_METHOD3(getFileBlockLocations, std::vector<Hdfs::BlockLocation> (const char * path, int64_t start, int64_t len));
MOCK_METHOD2(listAllDirectoryItems, std::vector<Hdfs::FileStatus> (const char * path, bool needLocation));
MOCK_METHOD0(getPeerCache, Hdfs::Internal::PeerCache &());
+ MOCK_METHOD2(createEncryptionZone, bool(const char * path, const char * keyName));
+ MOCK_METHOD1(getEZForPath, Hdfs::EncryptionZoneInfo(const char * path));
+ MOCK_METHOD2(listEncryptionZones, bool(const int64_t id, std::vector<Hdfs::EncryptionZoneInfo> &));
+ MOCK_METHOD0(listEncryptionZone, Hdfs::EncryptionZoneIterator());
+ MOCK_METHOD0(listAllEncryptionZoneItems, std::vector<Hdfs::EncryptionZoneInfo>());
};
#endif /* _HDFS_LIBHDFS3_MOCK_MOCKSOCKET_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneInfo.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/EncryptionZoneInfo.h b/depends/libhdfs3/src/client/EncryptionZoneInfo.h
new file mode 100644
index 0000000..d436ae7
--- /dev/null
+++ b/depends/libhdfs3/src/client/EncryptionZoneInfo.h
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _HDFS_LIBHDFS3_CLIENT_ENCRYPTIONZONEINFO_H_
+#define _HDFS_LIBHDFS3_CLIENT_ENCRYPTIONZONEINFO_H_
+
+#include <string>
+
+namespace Hdfs {
+
+class EncryptionZoneInfo {
+public:
+ EncryptionZoneInfo() :
+ suite(0), cryptoProtocolVersion(0), id(0) {
+ }
+
+ int getSuite() const {
+ return suite;
+ }
+
+ void setSuite(int suite) {
+ this->suite = suite;
+ }
+
+ int getCryptoProtocolVersion() const {
+ return cryptoProtocolVersion;
+ }
+
+ void setCryptoProtocolVersion(int cryptoProtocolVersion) {
+ this->cryptoProtocolVersion = cryptoProtocolVersion;
+ }
+
+ int getId() const {
+ return id;
+ }
+
+ void setId(int id) {
+ this->id = id;
+ }
+
+ const char * getPath() const{
+ return path.c_str();
+ }
+
+ void setPath(const char * path){
+ this->path = path;
+ }
+
+ const char * getKeyName() const{
+ return keyName.c_str();
+ }
+
+ void setKeyName(const char * keyName){
+ this->keyName = keyName;
+ }
+
+private:
+ int suite;
+ int cryptoProtocolVersion;
+ int64_t id;
+ std::string path;
+ std::string keyName;
+};
+
+}
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp b/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp
new file mode 100644
index 0000000..085541a
--- /dev/null
+++ b/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp
@@ -0,0 +1,86 @@
+/********************************************************************
+ * 2014 -
+ * open source under Apache License Version 2.0
+ ********************************************************************/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "EncryptionZoneIterator.h"
+#include "Exception.h"
+#include "ExceptionInternal.h"
+#include "EncryptionZoneInfo.h"
+#include "FileSystemImpl.h"
+
+namespace Hdfs {
+EncryptionZoneIterator::EncryptionZoneIterator() :filesystem(NULL), id(0), next(0) {
+}
+
+EncryptionZoneIterator::EncryptionZoneIterator(Hdfs::Internal::FileSystemImpl * const fs,
+ const int64_t id) :filesystem(fs), id(id), next(0) {
+}
+
+EncryptionZoneIterator::EncryptionZoneIterator(const EncryptionZoneIterator & it) :
+ filesystem(it.filesystem), id(it.id), next(it.next), lists(it.lists) {
+}
+
+EncryptionZoneIterator & EncryptionZoneIterator::operator =(const EncryptionZoneIterator & it) {
+ if (this == &it) {
+ return *this;
+ }
+
+ filesystem = it.filesystem;
+ id = it.id;
+ next = it.next;
+ lists = it.lists;
+ return *this;
+}
+
+bool EncryptionZoneIterator::listEncryptionZones() {
+ bool more;
+
+ if (NULL == filesystem) {
+ return false;
+ }
+
+ next = 0;
+ lists.clear();
+ more = filesystem->listEncryptionZones(id, lists);
+ if (!lists.empty()){
+ id = lists.back().getId();
+ }
+
+ return more || !lists.empty();
+}
+
+bool EncryptionZoneIterator::hasNext() {
+ if (next >= lists.size()) {
+ return listEncryptionZones();
+ }
+
+ return true;
+}
+
+Hdfs::EncryptionZoneInfo EncryptionZoneIterator::getNext() {
+ if (next >= lists.size()) {
+ if (!listEncryptionZones()) {
+ THROW(HdfsIOException, "End of the dir flow");
+ }
+ }
+ return lists[next++];
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneIterator.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/EncryptionZoneIterator.h b/depends/libhdfs3/src/client/EncryptionZoneIterator.h
new file mode 100644
index 0000000..9e37559
--- /dev/null
+++ b/depends/libhdfs3/src/client/EncryptionZoneIterator.h
@@ -0,0 +1,56 @@
+/********************************************************************
+ * 2014 -
+ * open source under Apache License Version 2.0
+ ********************************************************************/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_
+#define _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_
+
+#include "FileStatus.h"
+#include "EncryptionZoneInfo.h"
+#include <vector>
+
+namespace Hdfs {
+namespace Internal {
+class FileSystemImpl;
+}
+
+class EncryptionZoneIterator {
+public:
+ EncryptionZoneIterator();
+ EncryptionZoneIterator(Hdfs::Internal::FileSystemImpl * const fs,
+ const int64_t id);
+ EncryptionZoneIterator(const EncryptionZoneIterator & it);
+ EncryptionZoneIterator & operator = (const EncryptionZoneIterator & it);
+ bool hasNext();
+ EncryptionZoneInfo getNext();
+
+private:
+ bool listEncryptionZones();
+
+private:
+ Hdfs::Internal::FileSystemImpl * filesystem;
+ int64_t id;
+ size_t next;
+ std::vector<EncryptionZoneInfo> lists;
+};
+
+}
+
+#endif /* _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileEncryptionInfo.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileEncryptionInfo.h b/depends/libhdfs3/src/client/FileEncryptionInfo.h
new file mode 100644
index 0000000..32ead6c
--- /dev/null
+++ b/depends/libhdfs3/src/client/FileEncryptionInfo.h
@@ -0,0 +1,93 @@
+/********************************************************************
+ * 2014 -
+ * open source under Apache License Version 2.0
+ ********************************************************************/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _HDFS_LIBHDFS3_CLIENT_FILEENCRYPTIONINFO_H_
+#define _HDFS_LIBHDFS3_CLIENT_FILEENCRYPTIONINFO_H_
+
+#include <string>
+
+namespace Hdfs {
+
+class FileEncryptionInfo {
+public:
+ FileEncryptionInfo() :
+ cryptoProtocolVersion(0), suite(0){
+ }
+
+ int getSuite() const {
+ return suite;
+ }
+
+ void setSuite(int suite) {
+ this->suite = suite;
+ }
+
+ int getCryptoProtocolVersion() const {
+ return cryptoProtocolVersion;
+ }
+
+ void setCryptoProtocolVersion(int cryptoProtocolVersion) {
+ this->cryptoProtocolVersion = cryptoProtocolVersion;
+ }
+
+ const std::string & getKey() const{
+ return key;
+ }
+
+ void setKey(const std::string & key){
+ this->key = key;
+ }
+
+ const std::string & getKeyName() const{
+ return keyName;
+ }
+
+ void setKeyName(const std::string & keyName){
+ this->keyName = keyName;
+ }
+
+ const std::string & getIv() const{
+ return iv;
+ }
+
+ void setIv(const std::string & iv){
+ this->iv = iv;
+ }
+
+ const std::string & getEzKeyVersionName() const{
+ return ezKeyVersionName;
+ }
+
+ void setEzKeyVersionName(const std::string & ezKeyVersionName){
+ this->ezKeyVersionName = ezKeyVersionName;
+ }
+
+private:
+ int suite;
+ int cryptoProtocolVersion;
+ std::string key;
+ std::string iv;
+ std::string keyName;
+ std::string ezKeyVersionName;
+};
+
+}
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileStatus.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileStatus.h b/depends/libhdfs3/src/client/FileStatus.h
index 51b5096..1033b80 100644
--- a/depends/libhdfs3/src/client/FileStatus.h
+++ b/depends/libhdfs3/src/client/FileStatus.h
@@ -23,8 +23,9 @@
#define _HDFS_LIBHDFS3_CLIENT_FILESTATUS_H_
#include "Permission.h"
+#include "client/FileEncryptionInfo.h"
-#include <string>
+#include <string.h>
namespace Hdfs {
@@ -143,6 +144,21 @@ public:
return !symlink.empty();
}
+ /**
+ * Get encryption information for a file.
+ */
+ FileEncryptionInfo* getFileEncryption(){
+ return &fileEncryption;
+ }
+
+ /**
+ * Is an encryption file?
+ * @return true is this is an encryption file
+ */
+ bool isFileEncrypted() const {
+ return fileEncryption.getKey().length() > 0 && fileEncryption.getKeyName().length() > 0;
+ }
+
private:
bool isdir;
int64_t atime;
@@ -155,6 +171,7 @@ private:
std::string owner;
std::string path;
std::string symlink;
+ FileEncryptionInfo fileEncryption;
};
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystem.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileSystem.cpp b/depends/libhdfs3/src/client/FileSystem.cpp
index 6a660a4..8c18590 100644
--- a/depends/libhdfs3/src/client/FileSystem.cpp
+++ b/depends/libhdfs3/src/client/FileSystem.cpp
@@ -20,6 +20,7 @@
* limitations under the License.
*/
#include "DirectoryIterator.h"
+#include "EncryptionZoneIterator.h"
#include "Exception.h"
#include "ExceptionInternal.h"
#include "FileSystem.h"
@@ -582,4 +583,57 @@ void FileSystem::cancelDelegationToken(const std::string & token) {
impl->filesystem->cancelDelegationToken(token);
}
+
+/**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ */
+bool FileSystem::createEncryptionZone(const char * path, const char * keyName) {
+ if (!impl) {
+ THROW(HdfsIOException, "FileSystem: not connected.");
+ }
+
+ return impl->filesystem->createEncryptionZone(path, keyName);
+}
+
+/**
+* To get encryption zone information.
+* @param path the path which information is to be returned.
+* @return the encryption zone information.
+*/
+EncryptionZoneInfo FileSystem::getEZForPath(const char * path) {
+ if (!impl) {
+ THROW(HdfsIOException, "FileSystem: not connected.");
+ }
+
+ return impl->filesystem->getEZForPath(path);
+}
+
+/**
+ * list the contents of an encryption zone.
+ * @return Return a iterator to visit all elements in this encryption zone.
+ */
+EncryptionZoneIterator FileSystem::listEncryptionZone() {
+ if (!impl) {
+ THROW(HdfsIOException, "FileSystem: not connected.");
+ }
+
+ return impl->filesystem->listEncryptionZone();
+}
+
+/**
+* list all the contents of encryption zones.
+* @param id the index of encryption zones.
+* @return Return a vector of encryption zones information..
+*/
+std::vector<EncryptionZoneInfo> FileSystem::listAllEncryptionZoneItems() {
+ if (!impl) {
+ THROW(HdfsIOException, "FileSystem: not connected.");
+ }
+
+ return impl->filesystem->listAllEncryptionZoneItems();
+}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystem.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileSystem.h b/depends/libhdfs3/src/client/FileSystem.h
index 4f99c3c..e1f4dd2 100644
--- a/depends/libhdfs3/src/client/FileSystem.h
+++ b/depends/libhdfs3/src/client/FileSystem.h
@@ -24,8 +24,10 @@
#include "BlockLocation.h"
#include "DirectoryIterator.h"
+#include "EncryptionZoneIterator.h"
#include "FileStatus.h"
#include "FileSystemStats.h"
+#include "EncryptionZoneInfo.h"
#include "Permission.h"
#include "XmlConfig.h"
@@ -276,6 +278,36 @@ public:
*/
void cancelDelegationToken(const std::string & token);
+ /**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ */
+ bool createEncryptionZone(const char * path, const char * keyName);
+
+ /**
+ * To get encryption zone information.
+ * @param path the path which information is to be returned.
+ * @return the encryption zone information.
+ */
+ EncryptionZoneInfo getEZForPath(const char * path);
+
+ /**
+ * list the contents of an encryption zone;
+ * @return Return a iterator to visit all elements in this encryption zone.
+ */
+ EncryptionZoneIterator listEncryptionZone();
+
+
+ /**
+ * list all the contents of encryption zones.
+ * @param id the index of encryption zones.
+ * @return Return a vector of encryption zones information..
+ */
+ std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems();
+
+
private:
Config conf;
Internal::FileSystemWrapper * impl;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemImpl.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileSystemImpl.cpp b/depends/libhdfs3/src/client/FileSystemImpl.cpp
index 7b0f20a..6ee2b91 100644
--- a/depends/libhdfs3/src/client/FileSystemImpl.cpp
+++ b/depends/libhdfs3/src/client/FileSystemImpl.cpp
@@ -22,11 +22,13 @@
#include "Atomic.h"
#include "BlockLocation.h"
#include "DirectoryIterator.h"
+#include "EncryptionZoneIterator.h"
#include "Exception.h"
#include "ExceptionInternal.h"
#include "FileStatus.h"
#include "FileSystemImpl.h"
#include "FileSystemStats.h"
+#include "EncryptionZoneInfo.h"
#include "InputStream.h"
#include "LeaseRenewer.h"
#include "Logger.h"
@@ -775,5 +777,91 @@ bool FileSystemImpl::unregisterOpenedOutputStream() {
return openedOutputStream == 0;
}
+/**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ */
+
+bool FileSystemImpl::createEncryptionZone(const char * path, const char * keyName) {
+ if (!nn) {
+ THROW(HdfsIOException, "FileSystemImpl: not connected.");
+ }
+
+ if (NULL == path || !strlen(path)) {
+ THROW(InvalidParameter, "Invalid input: path should not be empty");
+ }
+
+ if (NULL == keyName || !strlen(keyName)) {
+ THROW(InvalidParameter, "Invalid input: key name should not be empty");
+ }
+
+ return nn->createEncryptionZone(getStandardPath(path), keyName);
+}
+
+
+/**
+ * To get encryption zone information.
+ * @param path the path which information is to be returned.
+ * @return the encryption zone information.
+ */
+
+EncryptionZoneInfo FileSystemImpl::getEZForPath(const char * path) {
+ if (!nn) {
+ THROW(HdfsIOException, "FileSystemImpl: not connected.");
+ }
+
+ if (NULL == path || !strlen(path)) {
+ THROW(InvalidParameter, "Invalid input: path should not be empty");
+ }
+
+ return nn->getEncryptionZoneInfo(getStandardPath(path), NULL);
+}
+
+bool FileSystemImpl::listEncryptionZones(const int64_t id,
+ std::vector<EncryptionZoneInfo> & ezl) {
+ if (!nn) {
+ THROW(HdfsIOException, "FileSystemImpl: not connected.");
+ }
+
+ return nn->listEncryptionZones(id, ezl);
+}
+
+/**
+ * list the contents of an encryption zone.
+ * @return return the encryption zone information.
+ */
+EncryptionZoneIterator FileSystemImpl::listEncryptionZone() {
+ if (!nn) {
+ THROW(HdfsIOException, "FileSystemImpl: not connected.");
+ }
+
+ return EncryptionZoneIterator(this, 0);
+}
+/**
+ * list all the contents of encryption zones.
+ * @param id the index of the encyrption zones.
+ * @return Return a vector of encryption zones information.
+ */
+
+std::vector<EncryptionZoneInfo> FileSystemImpl::listAllEncryptionZoneItems() {
+ if (!nn) {
+ THROW(HdfsIOException, "FileSystemImpl: not connected.");
+ }
+
+ std::vector<EncryptionZoneInfo> retval;
+ retval.clear();
+ int64_t id = 0;
+
+ EncryptionZoneIterator it;
+ it = FileSystemImpl::listEncryptionZone();
+
+ while (it.hasNext()) {
+ retval.push_back(it.getNext());
+ }
+ return retval;
+}
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemImpl.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileSystemImpl.h b/depends/libhdfs3/src/client/FileSystemImpl.h
index bd590bd..1c11a61 100644
--- a/depends/libhdfs3/src/client/FileSystemImpl.h
+++ b/depends/libhdfs3/src/client/FileSystemImpl.h
@@ -24,10 +24,12 @@
#include "BlockLocation.h"
#include "DirectoryIterator.h"
+#include "EncryptionZoneIterator.h"
#include "FileStatus.h"
#include "FileSystemInter.h"
#include "FileSystemKey.h"
#include "FileSystemStats.h"
+#include "EncryptionZoneInfo.h"
#include "Permission.h"
#include "server/Namenode.h"
#include "SessionConfig.h"
@@ -477,6 +479,44 @@ public:
return *peerCache;
}
+ /**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ */
+ bool createEncryptionZone(const char * path, const char * keyName);
+
+ /**
+ * To get encryption zone information.
+ * @param path the path which information is to be returned.
+ * @return the encryption zone information.
+ */
+ EncryptionZoneInfo getEZForPath(const char * path);
+
+ /**
+ * Get a partial listing of the indicated encryption zones
+ *
+ * @param id the index of encryption zones.
+ * @param ezl append the returned encryption zones.
+ * @return return true if there are more items.
+ */
+ bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl);
+
+ /**
+ * list the contents of an encryption zone.
+ * @return Return a iterator to visit all elements in this encryption zone.
+ */
+ EncryptionZoneIterator listEncryptionZone();
+
+
+ /**
+ * list all the contents of encryption zones.
+ * @param id the index of encryption zones.
+ * @return Return a vector of encryption zones information..
+ */
+ std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems();
+
private:
Config conf;
FileSystemKey key;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemInter.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/FileSystemInter.h b/depends/libhdfs3/src/client/FileSystemInter.h
index 352c37f..45df347 100644
--- a/depends/libhdfs3/src/client/FileSystemInter.h
+++ b/depends/libhdfs3/src/client/FileSystemInter.h
@@ -27,9 +27,11 @@
#include "BlockLocation.h"
#include "DirectoryIterator.h"
+#include "EncryptionZoneIterator.h"
#include "FileStatus.h"
#include "FileSystemKey.h"
#include "FileSystemStats.h"
+#include "EncryptionZoneInfo.h"
#include "PeerCache.h"
#include "Permission.h"
#include "server/LocatedBlocks.h"
@@ -481,6 +483,44 @@ public:
* @return return the peer cache.
*/
virtual PeerCache& getPeerCache() = 0;
+
+ /**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ */
+ virtual bool createEncryptionZone(const char * path, const char * keyName) = 0;
+
+ /**
+ * To get encryption zone information.
+ * @param path the path which information is to be returned.
+ * @return the encryption zone information.
+ */
+ virtual EncryptionZoneInfo getEZForPath(const char * path) = 0;
+
+ /**
+ * Get a partial listing of the indicated encryption zones
+ *
+ * @param id the index of encryption zones.
+ * @param ezl append the returned encryption zones.
+ * @return return true if there are more items.
+ */
+ virtual bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) = 0;
+
+ /**
+ * list the contents of an encryption zone.
+ * @return Return a iterator to visit all elements in this encryption zone.
+ */
+ virtual EncryptionZoneIterator listEncryptionZone() = 0;
+
+
+ /**
+ * list all the contents of encryption zones.
+ * @param id the index of encryption zones.
+ * @return Return a vector of encryption zones information..
+ */
+ virtual std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems() = 0;
};
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/Hdfs.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/Hdfs.cpp b/depends/libhdfs3/src/client/Hdfs.cpp
index 395f4f8..2840adc 100644
--- a/depends/libhdfs3/src/client/Hdfs.cpp
+++ b/depends/libhdfs3/src/client/Hdfs.cpp
@@ -989,8 +989,31 @@ int hdfsSetReplication(hdfsFS fs, const char * path, int16_t replication) {
return -1;
}
+static void ConstructHdfsEncryptionZoneInfo(hdfsEncryptionZoneInfo * infoEn,
+ std::vector<Hdfs::EncryptionZoneInfo> & enStatus) {
+ size_t size = enStatus.size();
+
+ for (size_t i = 0; i < size; ++i) {
+ infoEn[i].mSuite = enStatus[i].getSuite();
+ infoEn[i].mCryptoProtocolVersion = enStatus[i].getCryptoProtocolVersion();
+ infoEn[i].mId = enStatus[i].getId();
+ infoEn[i].mPath = Strdup(enStatus[i].getPath());
+ infoEn[i].mKeyName = Strdup(enStatus[i].getKeyName());
+ }
+}
+
+static void ConstructHdfsEncryptionFileInfo(hdfsEncryptionFileInfo * infoEn,
+ Hdfs::FileEncryptionInfo* enStatus) {
+ infoEn->mSuite = enStatus->getSuite();
+ infoEn->mCryptoProtocolVersion = enStatus->getCryptoProtocolVersion();
+ infoEn->mKey = const_cast<char*>(enStatus->getKey().c_str());
+ infoEn->mKeyName = const_cast<char*>(enStatus->getKeyName().c_str());
+ infoEn->mIv = const_cast<char*>(enStatus->getIv().c_str());
+ infoEn->mEzKeyVersionName = const_cast<char*>(enStatus->getEzKeyVersionName().c_str());
+}
+
static void ConstructHdfsFileInfo(hdfsFileInfo * infos,
- const std::vector<Hdfs::FileStatus> & status) {
+ std::vector<Hdfs::FileStatus> & status) {
size_t size = status.size();
for (size_t i = 0; i < size; ++i) {
@@ -1006,6 +1029,13 @@ static void ConstructHdfsFileInfo(hdfsFileInfo * infos,
infos[i].mPermissions = status[i].getPermission().toShort();
infos[i].mReplication = status[i].getReplication();
infos[i].mSize = status[i].getLength();
+ infos[i].mHdfsEncryptionFileInfo = NULL;
+ if (status[i].isFileEncrypted()) {
+ infos[i].mHdfsEncryptionFileInfo = new hdfsEncryptionFileInfo[1];
+ memset(infos[i].mHdfsEncryptionFileInfo, 0, sizeof(hdfsEncryptionFileInfo));
+ ConstructHdfsEncryptionFileInfo(infos[i].mHdfsEncryptionFileInfo, status[i].getFileEncryption());
+
+ }
}
}
@@ -1021,7 +1051,7 @@ hdfsFileInfo * hdfsListDirectory(hdfsFS fs, const char * path,
size = status.size();
retval = new hdfsFileInfo[size];
memset(retval, 0, sizeof(hdfsFileInfo) * size);
- ConstructHdfsFileInfo(&retval[0], status);
+ ConstructHdfsFileInfo(retval, status);
*numEntries = size;
return retval;
} catch (const std::bad_alloc & e) {
@@ -1061,11 +1091,22 @@ hdfsFileInfo * hdfsGetPathInfo(hdfsFS fs, const char * path) {
return NULL;
}
+void hdfsFreeEncryptionZoneInfo(hdfsEncryptionZoneInfo * infos, int numEntries) {
+ for (int i = 0; infos != NULL && i < numEntries; ++i) {
+ delete [] infos[i].mPath;
+ delete [] infos[i].mKeyName;
+ }
+ delete[] infos;
+}
+
void hdfsFreeFileInfo(hdfsFileInfo * infos, int numEntries) {
for (int i = 0; infos != NULL && i < numEntries; ++i) {
delete [] infos[i].mGroup;
delete [] infos[i].mName;
delete [] infos[i].mOwner;
+ if (infos[i].mHdfsEncryptionFileInfo != NULL) {
+ delete [] infos[i].mHdfsEncryptionFileInfo;
+ }
}
delete[] infos;
@@ -1450,6 +1491,75 @@ void hdfsFreeFileBlockLocations(BlockLocation * locations, int numOfBlock) {
delete [] locations;
}
+int hdfsCreateEncryptionZone(hdfsFS fs, const char * path, const char * keyName) {
+ PARAMETER_ASSERT(fs && path && strlen(path) > 0 && keyName && strlen(keyName) > 0, -1, EINVAL);
+
+ try {
+ return fs->getFilesystem().createEncryptionZone(path, keyName) ? 0 : -1;
+ } catch (const std::bad_alloc & e) {
+ SetErrorMessage("Out of memory");
+ errno = ENOMEM;
+ } catch (...) {
+ SetLastException(Hdfs::current_exception());
+ handleException(Hdfs::current_exception());
+ }
+
+ return -1;
+}
+
+hdfsEncryptionZoneInfo * hdfsGetEZForPath(hdfsFS fs, const char * path) {
+ PARAMETER_ASSERT(fs && path && strlen(path) > 0, NULL, EINVAL);
+ hdfsEncryptionZoneInfo * retval = NULL;
+
+ try {
+ retval = new hdfsEncryptionZoneInfo[1];
+ memset(retval, 0, sizeof(hdfsEncryptionZoneInfo));
+ std::vector<Hdfs::EncryptionZoneInfo> enStatus(1);
+ enStatus[0] = fs->getFilesystem().getEZForPath(path);
+ ConstructHdfsEncryptionZoneInfo(retval, enStatus);
+ return retval;
+ } catch (const std::bad_alloc & e) {
+ SetErrorMessage("Out of memory");
+ hdfsFreeEncryptionZoneInfo(retval, 1);
+ /* If out of memory error occurred, free hdfsEncryptionZoneInfo array's memory. */
+ errno = ENOMEM;
+ } catch (...) {
+ SetLastException(Hdfs::current_exception());
+ hdfsFreeEncryptionZoneInfo(retval, 1);
+ /* If any exceptions throw out, free hdfsEncryptionZoneInfo array's memory. */
+ handleException(Hdfs::current_exception());
+ }
+
+ return NULL;
+}
+
+
+hdfsEncryptionZoneInfo * hdfsListEncryptionZones(hdfsFS fs, int * numEntries) {
+ PARAMETER_ASSERT(fs, NULL, EINVAL);
+ hdfsEncryptionZoneInfo * retval = NULL;
+ int size = 0;
+
+ try {
+ std::vector<Hdfs::EncryptionZoneInfo> enStatus =
+ fs->getFilesystem().listAllEncryptionZoneItems();
+ size = enStatus.size();
+ retval = new hdfsEncryptionZoneInfo[size];
+ memset(retval, 0, sizeof(hdfsEncryptionZoneInfo) * size);
+ ConstructHdfsEncryptionZoneInfo(&retval[0], enStatus);
+ *numEntries = size;
+ return retval;
+ } catch (const std::bad_alloc & e) {
+ SetErrorMessage("Out of memory");
+ /* If out of memory error occurred, free hdfsEncryptionZoneInfo array's memory. */
+ hdfsFreeEncryptionZoneInfo(retval, size);
+ } catch (...) {
+ SetLastException(Hdfs::current_exception());
+ /* If any exceptions throw out, free hdfsEncryptionZoneInfo array's memory. */
+ hdfsFreeEncryptionZoneInfo(retval, size);
+ handleException(Hdfs::current_exception());
+ }
+ return NULL;
+}
#ifdef __cplusplus
}
#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/hdfs.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/hdfs.h b/depends/libhdfs3/src/client/hdfs.h
index 5d356f9..f8b61ea 100644
--- a/depends/libhdfs3/src/client/hdfs.h
+++ b/depends/libhdfs3/src/client/hdfs.h
@@ -482,6 +482,30 @@ int hdfsCreateDirectory(hdfsFS fs, const char * path);
int hdfsSetReplication(hdfsFS fs, const char * path, int16_t replication);
/**
+ * hdfsEncryptionZoneInfo- Information about an encryption zone.
+ */
+typedef struct {
+ int mSuite; /* the suite of encryption zone */
+ int mCryptoProtocolVersion; /* the version of crypto protocol */
+ int64_t mId; /* the id of encryption zone */
+ char * mPath; /* the path of encryption zone */
+ char * mKeyName; /* the key name of encryption zone */
+} hdfsEncryptionZoneInfo;
+
+
+/**
+ * hdfsEncryptionFileInfo - Information about an encryption file/directory.
+ */
+typedef struct {
+ int mSuite; /* the suite of encryption file/directory */
+ int mCryptoProtocolVersion; /* the version of crypto protocol */
+ char * mKey; /* the key of encryption file/directory */
+ char * mKeyName; /* the key name of encryption file/directory */
+ char * mIv; /* the iv of encryption file/directory */
+ char * mEzKeyVersionName; /* the version encryption file/directory */
+} hdfsEncryptionFileInfo;
+
+/**
* hdfsFileInfo - Information about a file/directory.
*/
typedef struct {
@@ -495,6 +519,7 @@ typedef struct {
char * mGroup; /* the group associated with the file */
short mPermissions; /* the permissions associated with the file */
tTime mLastAccess; /* the last access time for the file in seconds */
+ hdfsEncryptionFileInfo * mHdfsEncryptionFileInfo; /* the encryption info of the file/directory */
} hdfsFileInfo;
/**
@@ -528,6 +553,15 @@ hdfsFileInfo * hdfsGetPathInfo(hdfsFS fs, const char * path);
void hdfsFreeFileInfo(hdfsFileInfo * infos, int numEntries);
/**
+ * hdfsFreeEncryptionZoneInfo - Free up the hdfsEncryptionZoneInfo array (including fields)
+ * @param infos The array of dynamically-allocated hdfsEncryptionZoneInfo
+ * objects.
+ * @param numEntries The size of the array.
+ */
+void hdfsFreeEncryptionZoneInfo(hdfsEncryptionZoneInfo * infos, int numEntries);
+
+
+/**
* hdfsGetHosts - Get hostnames where a particular block (determined by
* pos & blocksize) of a file is stored. The last element in the array
* is NULL. Due to replication, a single block could be present on
@@ -723,6 +757,35 @@ BlockLocation * hdfsGetFileBlockLocations(hdfsFS fs, const char * path,
*/
void hdfsFreeFileBlockLocations(BlockLocation * locations, int numOfBlock);
+/**
+ * Create encryption zone for the directory with specific key name
+ * @param fs The configured filesystem handle.
+ * @param path The path of the directory.
+ * @param keyname The key name of the encryption zone
+ * @return Returns 0 on success, -1 on error.
+ */
+int hdfsCreateEncryptionZone(hdfsFS fs, const char * path, const char * keyName);
+
+/**
+ * hdfsEncryptionZoneInfo - Get information about a path as a (dynamically
+ * allocated) single hdfsEncryptionZoneInfo struct. hdfsEncryptionZoneInfo should be
+ * called when the pointer is no longer needed.
+ * @param fs The configured filesystem handle.
+ * @param path The path of the encryption zone.
+ * @return Returns a dynamically-allocated hdfsEncryptionZoneInfo object;
+ * NULL on error.
+ */
+hdfsEncryptionZoneInfo * hdfsGetEZForPath(hdfsFS fs, const char * path);
+
+/**
+ * hdfsEncryptionZoneInfo - Get list of all the encryption zones.
+ * hdfsFreeEncryptionZoneInfo should be called to deallocate memory.
+ * @param fs The configured filesystem handle.
+ * @return Returns a dynamically-allocated array of hdfsEncryptionZoneInfo objects;
+ * NULL on error.
+ */
+hdfsEncryptionZoneInfo * hdfsListEncryptionZones(hdfsFS fs, int * numEntries);
+
#ifdef __cplusplus
}
#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto b/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto
index 5362246..1ab69cd 100644
--- a/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto
+++ b/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto
@@ -33,6 +33,7 @@ package Hdfs.Internal;
import "hdfs.proto";
import "Security.proto";
+import "encryption.proto";
/**
* The ClientNamenodeProtocol Service defines the interface between a client
@@ -74,6 +75,7 @@ message CreateRequestProto {
required bool createParent = 5;
required uint32 replication = 6; // Short: Only 16 bits used
required uint64 blockSize = 7;
+ repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8;
}
message CreateResponseProto {
@@ -752,4 +754,10 @@ service ClientNamenodeProtocol {
returns(GetSnapshotDiffReportResponseProto);
rpc isFileClosed(IsFileClosedRequestProto)
returns(IsFileClosedResponseProto);
+ rpc createEncryptionZone(CreateEncryptionZoneRequestProto)
+ returns(CreateEncryptionZoneResponseProto);
+ rpc listEncryptionZones(ListEncryptionZonesRequestProto)
+ returns(ListEncryptionZonesResponseProto);
+ rpc getEZForPath(GetEZForPathRequestProto)
+ returns(GetEZForPathResponseProto);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/datatransfer.proto
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/proto/datatransfer.proto b/depends/libhdfs3/src/proto/datatransfer.proto
index 5d8013e..d787d8a 100644
--- a/depends/libhdfs3/src/proto/datatransfer.proto
+++ b/depends/libhdfs3/src/proto/datatransfer.proto
@@ -43,6 +43,7 @@ message DataTransferEncryptorMessageProto {
required DataTransferEncryptorStatus status = 1;
optional bytes payload = 2;
optional string message = 3;
+ repeated CipherOptionProto cipherOption = 4;
}
message BaseHeaderProto {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/encryption.proto
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/proto/encryption.proto b/depends/libhdfs3/src/proto/encryption.proto
new file mode 100644
index 0000000..53206f8
--- /dev/null
+++ b/depends/libhdfs3/src/proto/encryption.proto
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and stable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *stable* .proto interface.
+ */
+
+/** This file contains protocol buffers that are used throughout HDFS -- i.e.
+ * by the client, server, and data transfer protocols.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.protocol.proto";
+option java_outer_classname = "EncryptionZonesProtos";
+option java_generate_equals_and_hash = true;
+package Hdfs.Internal;
+
+import "hdfs.proto";
+
+message CreateEncryptionZoneRequestProto {
+ required string src = 1;
+ optional string keyName = 2;
+}
+
+message CreateEncryptionZoneResponseProto {
+}
+
+message ListEncryptionZonesRequestProto {
+ required int64 id = 1;
+}
+
+message EncryptionZoneProto {
+ required int64 id = 1;
+ required string path = 2;
+ required CipherSuiteProto suite = 3;
+ required CryptoProtocolVersionProto cryptoProtocolVersion = 4;
+ required string keyName = 5;
+}
+
+message ListEncryptionZonesResponseProto {
+ repeated EncryptionZoneProto zones = 1;
+ required bool hasMore = 2;
+}
+
+message GetEZForPathRequestProto {
+ required string src = 1;
+}
+
+message GetEZForPathResponseProto {
+ optional EncryptionZoneProto zone = 1;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/proto/hdfs.proto b/depends/libhdfs3/src/proto/hdfs.proto
index 19e3f79..d85f217 100644
--- a/depends/libhdfs3/src/proto/hdfs.proto
+++ b/depends/libhdfs3/src/proto/hdfs.proto
@@ -160,6 +160,64 @@ message DataEncryptionKeyProto {
optional string encryptionAlgorithm = 6;
}
+/**
+ * Cipher suite.
+ */
+enum CipherSuiteProto {
+ UNKNOWN = 1;
+ AES_CTR_NOPADDING = 2;
+}
+
+/**
+ * Crypto protocol version used to access encrypted files.
+ */
+enum CryptoProtocolVersionProto {
+ UNKNOWN_PROTOCOL_VERSION = 1;
+ ENCRYPTION_ZONES = 2;
+}
+
+/**
+ * Encryption information for a file.
+ */
+message FileEncryptionInfoProto {
+ required CipherSuiteProto suite = 1;
+ required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
+ required bytes key = 3;
+ required bytes iv = 4;
+ required string keyName = 5;
+ required string ezKeyVersionName = 6;
+}
+
+/**
+ * Encryption information for an individual
+ * file within an encryption zone
+ */
+message PerFileEncryptionInfoProto {
+ required bytes key = 1;
+ required bytes iv = 2;
+ required string ezKeyVersionName = 3;
+}
+
+/**
+ * Encryption information for an encryption
+ * zone
+ */
+message ZoneEncryptionInfoProto {
+ required CipherSuiteProto suite = 1;
+ required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
+ required string keyName = 3;
+}
+
+/**
+ * Cipher option
+ */
+message CipherOptionProto {
+ required CipherSuiteProto suite = 1;
+ optional bytes inKey = 2;
+ optional bytes inIv = 3;
+ optional bytes outKey = 4;
+ optional bytes outIv = 5;
+}
/**
* A set of file blocks and their locations.
@@ -203,6 +261,8 @@ message HdfsFileStatusProto {
// Optional field for fileId
optional uint64 fileId = 13 [default = 0]; // default as an invalid id
optional int32 childrenNum = 14 [default = -1];
+ // Optional field for file encryption
+ optional FileEncryptionInfoProto fileEncryptionInfo = 15;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/rpc/RpcAuth.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/rpc/RpcAuth.h b/depends/libhdfs3/src/rpc/RpcAuth.h
index df503bb..5075f08 100644
--- a/depends/libhdfs3/src/rpc/RpcAuth.h
+++ b/depends/libhdfs3/src/rpc/RpcAuth.h
@@ -33,7 +33,7 @@ namespace Internal {
enum AuthMethod {
SIMPLE = 80, KERBEROS = 81, //"GSSAPI"
TOKEN = 82, //"DIGEST-MD5"
- UNKNOWN = 255
+ UNSURENESS = 255
};
enum AuthProtocol {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/rpc/RpcChannel.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/rpc/RpcChannel.cpp b/depends/libhdfs3/src/rpc/RpcChannel.cpp
index 974e86c..7f9ef5d 100644
--- a/depends/libhdfs3/src/rpc/RpcChannel.cpp
+++ b/depends/libhdfs3/src/rpc/RpcChannel.cpp
@@ -121,7 +121,7 @@ const RpcSaslProto_SaslAuth * RpcChannelImpl::createSaslClient(
break;
} else if (method.getMethod() == AuthMethod::SIMPLE) {
return auth;
- } else if (method.getMethod() == AuthMethod::UNKNOWN) {
+ } else if (method.getMethod() == AuthMethod::UNSURENESS) {
return auth;
} else {
auth = NULL;
@@ -187,7 +187,7 @@ RpcAuth RpcChannelImpl::setupSaslConnection() {
if (retval.getMethod() == AuthMethod::SIMPLE) {
done = true;
- } else if (retval.getMethod() == AuthMethod::UNKNOWN) {
+ } else if (retval.getMethod() == AuthMethod::UNSURENESS) {
THROW(AccessControlException, "Unknown auth mechanism");
} else {
std::string respToken;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/Namenode.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/Namenode.h b/depends/libhdfs3/src/server/Namenode.h
index 9577b6d..ad213e8 100644
--- a/depends/libhdfs3/src/server/Namenode.h
+++ b/depends/libhdfs3/src/server/Namenode.h
@@ -23,6 +23,7 @@
#define _HDFS_LIBHDFS3_SERVER_NAMENODE_H_
#include "client/FileStatus.h"
+#include "client/EncryptionZoneInfo.h"
#include "client/Permission.h"
#include "DatanodeInfo.h"
#include "Exception.h"
@@ -809,8 +810,40 @@ public:
* close the namenode connection.
*/
virtual void close() {};
-};
+ /**
+ * Create encryption zone for the directory with specific key name
+ * @param path the directory path which is to be created.
+ * @param keyname The key name of the encryption zone
+ * @return return true if success.
+ * @throw HdfsIOException If an I/O error occurred
+ */
+ virtual bool createEncryptionZone(const std::string & src, const std::string & keyName) = 0;
+
+ /**
+ * To get encryption zone information.
+ * @param path the path which information is to be returned.
+ * @return the encryption zone information.
+ * @throw FileNotFoundException If file <code>src</code> does not exist
+ * @throw UnresolvedLinkException If <code>src</code> contains a symlink
+ * @throw HdfsIOException If an I/O error occurred
+ */
+ virtual EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist) = 0;
+
+ /**
+ * Get a partial listing of the indicated encryption zones
+ *
+ * @param id the index of encryption zone
+ * @param ezl append the returned encryption zones.
+ *
+ * @throw AccessControlException permission denied
+ * @throw UnresolvedLinkException If <code>src</code> contains a symlink
+ * @throw HdfsIOException If an I/O error occurred
+ */
+ virtual bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl)
+ /* throw (AccessControlException, UnresolvedLinkException, HdfsIOException) */ = 0;
+
+};
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeImpl.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/NamenodeImpl.cpp b/depends/libhdfs3/src/server/NamenodeImpl.cpp
index ae55e9d..958f6b1 100644
--- a/depends/libhdfs3/src/server/NamenodeImpl.cpp
+++ b/depends/libhdfs3/src/server/NamenodeImpl.cpp
@@ -85,6 +85,7 @@ void NamenodeImpl::create(const std::string & src, const Permission & masked,
FileAlreadyExistsException, FileNotFoundException,
NSQuotaExceededException, ParentNotDirectoryException,
UnresolvedLinkException, HdfsIOException) */{
+
try {
CreateRequestProto request;
CreateResponseProto response;
@@ -94,6 +95,7 @@ void NamenodeImpl::create(const std::string & src, const Permission & masked,
request.set_createparent(createParent);
request.set_replication(replication);
request.set_src(src);
+ request.add_cryptoprotocolversion(CryptoProtocolVersionProto::ENCRYPTION_ZONES);
Build(masked, request.mutable_masked());
invoke(RpcCall(false, "create", &request, &response));
} catch (const HdfsRpcServerException & e) {
@@ -792,5 +794,76 @@ void NamenodeImpl::cancelDelegationToken(const Token & token) {
}
}
+bool NamenodeImpl::createEncryptionZone(const std::string & src, const std::string & keyName) {
+ try {
+ CreateEncryptionZoneRequestProto request;
+ CreateEncryptionZoneResponseProto response;
+ request.set_src(src);
+ request.set_keyname(keyName);
+ invoke(RpcCall(true, "createEncryptionZone",&request, &response));
+ return true;
+ } catch (const HdfsRpcServerException & e) {
+ UnWrapper < HdfsIOException > unwrapper(e);
+ unwrapper.unwrap(__FILE__, __LINE__);
+ }
+}
+
+EncryptionZoneInfo NamenodeImpl::getEncryptionZoneInfo(const std::string & src, bool *exist)
+/* throw (FileNotFoundException,
+ UnresolvedLinkException, HdfsIOException) */{
+ EncryptionZoneInfo retval;
+
+ try {
+ GetEZForPathRequestProto request;
+ GetEZForPathResponseProto response;
+ request.set_src(src);
+ invoke(RpcCall(true, "getEZForPath", &request, &response));
+
+ if (response.has_zone()) {
+ Convert(retval, response.zone());
+ retval.setPath(src.c_str());
+
+ if (exist) {
+ *exist = true;
+ }
+
+ return retval;
+ }
+
+ if (!exist) {
+ THROW(FileNotFoundException, "Path %s does not exist.", src.c_str());
+ }
+
+ *exist = false;
+ } catch (const HdfsRpcServerException & e) {
+ UnWrapper < FileNotFoundException,
+ UnresolvedLinkException, HdfsIOException > unwrapper(e);
+ unwrapper.unwrap(__FILE__, __LINE__);
+ }
+
+ return retval;
+}
+
+//Idempotent
+bool NamenodeImpl::listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl)
+ /* throw (AccessControlException,FileNotFoundException, UnresolvedLinkException, HdfsIOException) */{
+ try {
+ ListEncryptionZonesRequestProto request;
+ ListEncryptionZonesResponseProto response;
+ request.set_id(id);
+ invoke(RpcCall(true, "listEncryptionZones", &request, &response));
+
+ if (response.zones_size() != 0) {
+ Convert(ezl, response);
+ return response.hasmore();
+ }
+
+ } catch (const HdfsRpcServerException & e) {
+ UnWrapper < FileNotFoundException,
+ UnresolvedLinkException, HdfsIOException > unwrapper(e);
+ unwrapper.unwrap(__FILE__, __LINE__);
+ }
+}
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeImpl.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/NamenodeImpl.h b/depends/libhdfs3/src/server/NamenodeImpl.h
index 2d915da..1abe6d9 100644
--- a/depends/libhdfs3/src/server/NamenodeImpl.h
+++ b/depends/libhdfs3/src/server/NamenodeImpl.h
@@ -218,6 +218,15 @@ public:
void cancelDelegationToken(const Token & token)
/*throws IOException*/;
+ bool createEncryptionZone(const std::string & src, const std::string & keyName);
+ /* throws HdfsIOException If an I/O error occurred */
+
+ EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist);
+ /* throw (FileNotFoundException, UnresolvedLinkException, HdfsIOException) */
+ bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl);
+ /* throw (AccessControlException, UnresolvedLinkException, HdfsIOException) */
+
+
private:
void invoke(const RpcCall & call);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeProxy.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/NamenodeProxy.cpp b/depends/libhdfs3/src/server/NamenodeProxy.cpp
index 893b44f..81581ef 100644
--- a/depends/libhdfs3/src/server/NamenodeProxy.cpp
+++ b/depends/libhdfs3/src/server/NamenodeProxy.cpp
@@ -524,5 +524,29 @@ void NamenodeProxy::close() {
namenodes.clear();
}
+bool NamenodeProxy::createEncryptionZone(const std::string & src, const std::string & keyName) {
+ NAMENODE_HA_RETRY_BEGIN();
+ return namenode->createEncryptionZone(src, keyName);
+ NAMENODE_HA_RETRY_END();
+ assert(!"should not reach here");
+ return false;
+}
+
+EncryptionZoneInfo NamenodeProxy::getEncryptionZoneInfo(const std::string & src, bool *exist) {
+ NAMENODE_HA_RETRY_BEGIN();
+ return namenode->getEncryptionZoneInfo(src, exist);
+ NAMENODE_HA_RETRY_END();
+ assert(!"should not reach here");
+ return EncryptionZoneInfo();
+}
+
+bool NamenodeProxy::listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) {
+ NAMENODE_HA_RETRY_BEGIN();
+ return namenode->listEncryptionZones(id, ezl);
+ NAMENODE_HA_RETRY_END();
+ assert(!"should not reach here");
+ return false;
+}
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeProxy.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/NamenodeProxy.h b/depends/libhdfs3/src/server/NamenodeProxy.h
index 7d22713..c5b5142 100644
--- a/depends/libhdfs3/src/server/NamenodeProxy.h
+++ b/depends/libhdfs3/src/server/NamenodeProxy.h
@@ -139,6 +139,13 @@ public:
void close();
+ bool createEncryptionZone(const std::string & path, const std::string & keyName);
+
+ EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist);
+
+ bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl);
+
+
private:
shared_ptr<Namenode> getActiveNamenode(uint32_t & oldValue);
void failoverToNextNamenode(uint32_t oldValue);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/RpcHelper.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/server/RpcHelper.h b/depends/libhdfs3/src/server/RpcHelper.h
index c6d9a4e..571ffc0 100644
--- a/depends/libhdfs3/src/server/RpcHelper.h
+++ b/depends/libhdfs3/src/server/RpcHelper.h
@@ -23,6 +23,7 @@
#define _HDFS_LIBHDFS3_SERVER_RPCHELPER_H_
#include "client/FileStatus.h"
+#include "client/EncryptionZoneInfo.h"
#include "client/Permission.h"
#include "ClientDatanodeProtocol.pb.h"
#include "ClientNamenodeProtocol.pb.h"
@@ -182,6 +183,36 @@ static inline void Convert(const std::string & src, FileStatus & fs,
fs.setSymlink(proto.symlink().c_str());
fs.setPermission(Permission(proto.permission().perm()));
fs.setIsdir(proto.filetype() == HdfsFileStatusProto::IS_DIR);
+
+ if (proto.has_fileencryptioninfo()){
+ const FileEncryptionInfoProto &encrypt = proto.fileencryptioninfo();
+ FileEncryptionInfo* convert = fs.getFileEncryption();
+ convert->setSuite(encrypt.suite());
+ convert->setCryptoProtocolVersion(encrypt.cryptoprotocolversion());
+ convert->setKey(encrypt.key());
+ convert->setKeyName(encrypt.keyname());
+ convert->setIv(encrypt.iv());
+ convert->setEzKeyVersionName(encrypt.ezkeyversionname());
+ }
+}
+
+static inline void Convert(EncryptionZoneInfo & enZone,
+ const EncryptionZoneProto & proto) {
+ enZone.setSuite(proto.suite());
+ enZone.setCryptoProtocolVersion(proto.cryptoprotocolversion());
+ enZone.setId(proto.id());
+ enZone.setPath(proto.path().c_str());
+ enZone.setKeyName(proto.keyname().c_str());
+}
+
+static inline void Convert(std::vector<EncryptionZoneInfo> & ezl,
+ const ListEncryptionZonesResponseProto & proto) {
+ RepeatedPtrField<EncryptionZoneProto> ptrproto = proto.zones();
+ for (int i=0; i < ptrproto.size(); i++) {
+ EncryptionZoneInfo enZoneInfo;
+ Convert(enZoneInfo, ptrproto.Get(i));
+ ezl.push_back(enZoneInfo);
+ }
}
static inline void Convert(const std::string & src,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestCInterface.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/test/function/TestCInterface.cpp b/depends/libhdfs3/test/function/TestCInterface.cpp
index 2b81067..e45aaee 100644
--- a/depends/libhdfs3/test/function/TestCInterface.cpp
+++ b/depends/libhdfs3/test/function/TestCInterface.cpp
@@ -30,6 +30,9 @@
#include <fcntl.h>
#include <stdlib.h>
#include <limits>
+#include <stdlib.h>
+#include <sstream>
+#include <iostream>
using namespace Hdfs::Internal;
@@ -198,6 +201,45 @@ TEST(TestCInterfaceConnect, TestConnect_Success) {
ASSERT_EQ(hdfsDisconnect(fs), 0);
}
+TEST(TestCInterfaceTDE, DISABLED_TestCreateEnRPC_Success) {
+ hdfsFS fs = NULL;
+ hdfsEncryptionZoneInfo * enInfo = NULL;
+ char * uri = NULL;
+ setenv("LIBHDFS3_CONF", "function-test.xml", 1);
+ struct hdfsBuilder * bld = hdfsNewBuilder();
+ assert(bld != NULL);
+ hdfsBuilderSetNameNode(bld, "default");
+ fs = hdfsBuilderConnect(bld);
+ ASSERT_TRUE(fs != NULL);
+ system("hadoop fs -rmr /TDE");
+ system("hadoop key create keytde");
+ system("hadoop fs -mkdir /TDE");
+ ASSERT_EQ(0, hdfsCreateEncryptionZone(fs, "/TDE", "keytde"));
+ enInfo = hdfsGetEZForPath(fs, "/TDE");
+ ASSERT_TRUE(enInfo != NULL);
+ EXPECT_TRUE(enInfo->mKeyName != NULL);
+ std::cout << "----hdfsEncryptionZoneInfo----:" << " KeyName : " << enInfo->mKeyName << " Suite : " << enInfo->mSuite << " CryptoProtocolVersion : " << enInfo->mCryptoProtocolVersion << " Id : " << enInfo->mId << " Path : " << enInfo->mPath << std::endl;
+ hdfsFreeEncryptionZoneInfo(enInfo, 1);
+ for (int i = 0; i <= 201; i++){
+ std::stringstream newstr;
+ newstr << i;
+ std::string tde = "/TDE" + newstr.str();
+ std::string key = "keytde" + newstr.str();
+ std::string rmTde = "hadoop fs -rmr /TDE" + newstr.str();
+ std::string tdeKey = "hadoop key create keytde" + newstr.str();
+ std::string mkTde = "hadoop fs -mkdir /TDE" + newstr.str();
+ system(rmTde.c_str());
+ system(tdeKey.c_str());
+ system(mkTde.c_str());
+ ASSERT_EQ(0, hdfsCreateEncryptionZone(fs, tde.c_str(), key.c_str()));
+ }
+ hdfsEncryptionZoneInfo * enZoneInfos = NULL;
+ int num = 0;
+ hdfsListEncryptionZones(fs, &num);
+ EXPECT_EQ(num, 203);
+ ASSERT_EQ(hdfsDisconnect(fs), 0);
+ hdfsFreeBuilder(bld);
+}
TEST(TestErrorMessage, TestErrorMessage) {
EXPECT_NO_THROW(hdfsGetLastError());
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestFileSystem.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/test/function/TestFileSystem.cpp b/depends/libhdfs3/test/function/TestFileSystem.cpp
index 3191adb..b9d3f6b 100644
--- a/depends/libhdfs3/test/function/TestFileSystem.cpp
+++ b/depends/libhdfs3/test/function/TestFileSystem.cpp
@@ -147,6 +147,39 @@ TEST_F(TestFileSystem, listDirectory) {
ASSERT_THROW(it.getNext(), HdfsIOException);
}
+TEST_F(TestFileSystem, DISABLED_listEncryptionZone) {
+ fs->disconnect();
+ fs->connect();
+ const int dirs = 201;
+
+ for (int i = 0; i < dirs; i++){
+ std::stringstream newstr;
+ newstr << i;
+ std::string tde = "/TDE" + newstr.str();
+ std::string key = "keytde" + newstr.str();
+ std::string rmTde = "hadoop fs -rmr /TDE" + newstr.str();
+ std::string tdeKey = "hadoop key create keytde" + newstr.str();
+ std::string mkTde = "hadoop fs -mkdir /TDE" + newstr.str();
+ std::string tdeZone = "hdfs crypto -createZone -keyName " + key + "-path " + tde;
+ system(rmTde.c_str());
+ system(tdeKey.c_str());
+ system(mkTde.c_str());
+ system(tdeZone.c_str());
+ }
+
+ EncryptionZoneIterator it;
+ EXPECT_NO_THROW(it = fs->listEncryptionZone());
+ int count = 0;
+
+ while (it.hasNext()) {
+ count ++;
+ it.getNext();
+ }
+
+ ASSERT_EQ(dirs, count);
+ ASSERT_THROW(it.getNext(), HdfsIOException);
+}
+
TEST_F(TestFileSystem, setOwner) {
fs->disconnect();
ASSERT_THROW(fs->setOwner(BASE_DIR, "setOwner", ""), HdfsIOException);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestOutputStream.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/test/function/TestOutputStream.cpp b/depends/libhdfs3/test/function/TestOutputStream.cpp
index faf84e7..e57df34 100644
--- a/depends/libhdfs3/test/function/TestOutputStream.cpp
+++ b/depends/libhdfs3/test/function/TestOutputStream.cpp
@@ -517,6 +517,18 @@ TEST_F(TestOutputStream, TestOpenFileForWrite) {
}
+TEST_F(TestOutputStream, DISABLE_TestOpenFileForWriteTDE){
+ conf.set("output.default.packetsize", 1024);
+ fs = new FileSystem(conf);
+ fs->connect();
+ fs->mkdirs("/testTDE", 0755);
+ system("hadoop key create amy");
+ system("hdfs crypto -createZone -keyName amy -path /testTDE");
+ OutputStream other;
+ ASSERT_NO_THROW(other.open(*fs, "/testTDE/amy", Create | Append));
+ other.close();
+ fs->disconnect();
+}
TEST_F(TestOutputStream, TestWriteChunkPacket) {
//test create a file and write a block