You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/09/18 20:43:00 UTC
git commit: HDFS-7047. Expose FileStatus#isEncrypted in libhdfs
(cmccabe) (cherry picked from commit
570b8b468e1a75ad29c674699f0f4fe34d919ed9)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d24ef142b -> 2b1c9c621
HDFS-7047. Expose FileStatus#isEncrypted in libhdfs (cmccabe)
(cherry picked from commit 570b8b468e1a75ad29c674699f0f4fe34d919ed9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b1c9c62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b1c9c62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b1c9c62
Branch: refs/heads/branch-2
Commit: 2b1c9c62100454ec8b7a8cc0f8268d9c5ba4ee51
Parents: d24ef14
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Sep 18 11:42:23 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Sep 18 11:42:53 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 69 +++++++++++++++++++-
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 8 +++
.../main/native/libhdfs/test_libhdfs_threaded.c | 1 +
4 files changed, 77 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 32f1df3..bbd80b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -217,6 +217,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
+ HDFS-7047. Expose FileStatus#isEncrypted in libhdfs (cmccabe)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
index 1d0278a..7c87adf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
@@ -79,6 +79,15 @@ struct hdfsFile_internal {
int flags;
};
+#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1
+
+/**
+ * Extended file information.
+ */
+struct hdfsExtendedFileInfo {
+ int flags;
+};
+
int hdfsFileIsOpenForRead(hdfsFile file)
{
return (file->type == HDFS_STREAM_INPUT);
@@ -2836,9 +2845,39 @@ tOffset hdfsGetUsed(hdfsFS fs)
}
return jVal.j;
}
+
+/**
+ * We cannot add new fields to the hdfsFileInfo structure because it would break
+ * binary compatibility. The reason is because we return an array
+ * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of
+ * those structures would break all programs that relied on finding the second
+ * element in the array at <base_offset> + sizeof(struct hdfsFileInfo).
+ *
+ * So instead, we add the new fields to the hdfsExtendedFileInfo structure.
+ * This structure is contained in the mOwner string found inside the
+ * hdfsFileInfo. Specifically, the format of mOwner is:
+ *
+ * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure]
+ *
+ * The padding is added so that the hdfsExtendedFileInfo structure starts on an
+ * 8-byte boundary.
+ *
+ * @param str The string to locate the extended info in.
+ * @return The offset of the hdfsExtendedFileInfo structure.
+ */
+static size_t getExtendedFileInfoOffset(const char *str)
+{
+ int num_64_bit_words = ((strlen(str) + 1) + 7) / 8;
+ return num_64_bit_words * 8;
+}
+static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo)
+{
+ char *owner = fileInfo->mOwner;
+ return (struct hdfsExtendedFileInfo *)(owner +
+ getExtendedFileInfoOffset(owner));
+}
-
static jthrowable
getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
{
@@ -2852,6 +2891,8 @@ getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
const char *cPathName;
const char *cUserName;
const char *cGroupName;
+ struct hdfsExtendedFileInfo *extInfo;
+ size_t extOffset;
jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
HADOOP_STAT, "isDir", "()Z");
@@ -2926,9 +2967,24 @@ getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
jthr = getPendingExceptionAndClear(env);
goto done;
}
- fileInfo->mOwner = strdup(cUserName);
+ extOffset = getExtendedFileInfoOffset(cUserName);
+ fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo));
+ if (!fileInfo->mOwner) {
+ jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner");
+ goto done;
+ }
+ strcpy(fileInfo->mOwner, cUserName);
(*env)->ReleaseStringUTFChars(env, jUserName, cUserName);
-
+ extInfo = getExtendedFileInfo(fileInfo);
+ memset(extInfo, 0, sizeof(*extInfo));
+ jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+ HADOOP_STAT, "isEncrypted", "()Z");
+ if (jthr) {
+ goto done;
+ }
+ if (jVal.z == JNI_TRUE) {
+ extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED;
+ }
jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
"getGroup", "()Ljava/lang/String;");
if (jthr)
@@ -3174,6 +3230,13 @@ void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
free(hdfsFileInfo);
}
+int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo)
+{
+ struct hdfsExtendedFileInfo *extInfo;
+
+ extInfo = getExtendedFileInfo(fileInfo);
+ return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
index 270db81..0625da3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
@@ -592,6 +592,14 @@ extern "C" {
*/
void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
+ /**
+ * hdfsFileIsEncrypted: determine if a file is encrypted based on its
+ * hdfsFileInfo.
+ * @return -1 if there was an error (errno will be set), 0 if the file is
+ * not encrypted, 1 if the file is encrypted.
+ */
+ int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo);
+
/**
* hdfsGetHosts - Get hostnames where a particular block (determined by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
index cf605e3..64c1a8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
@@ -217,6 +217,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
+ EXPECT_ZERO(hdfsFileIsEncrypted(fileInfo));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2"));