You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/09/18 20:43:00 UTC

git commit: HDFS-7047. Expose FileStatus#isEncrypted in libhdfs (cmccabe) (cherry picked from commit 570b8b468e1a75ad29c674699f0f4fe34d919ed9)

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d24ef142b -> 2b1c9c621


HDFS-7047. Expose FileStatus#isEncrypted in libhdfs (cmccabe)
(cherry picked from commit 570b8b468e1a75ad29c674699f0f4fe34d919ed9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b1c9c62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b1c9c62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b1c9c62

Branch: refs/heads/branch-2
Commit: 2b1c9c62100454ec8b7a8cc0f8268d9c5ba4ee51
Parents: d24ef14
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Sep 18 11:42:23 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Sep 18 11:42:53 2014 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  2 +
 .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c  | 69 +++++++++++++++++++-
 .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h  |  8 +++
 .../main/native/libhdfs/test_libhdfs_threaded.c |  1 +
 4 files changed, 77 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 32f1df3..bbd80b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -217,6 +217,8 @@ Release 2.6.0 - UNRELEASED
 
     HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
 
+    HDFS-7047. Expose FileStatus#isEncrypted in libhdfs (cmccabe)
+
   OPTIMIZATIONS
 
     HDFS-6690. Deduplicate xattr names in memory. (wang)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
index 1d0278a..7c87adf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
@@ -79,6 +79,15 @@ struct hdfsFile_internal {
     int flags;
 };
 
+#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1
+
+/**
+ * Extended file information.
+ */
+struct hdfsExtendedFileInfo {
+    int flags;
+};
+
 int hdfsFileIsOpenForRead(hdfsFile file)
 {
     return (file->type == HDFS_STREAM_INPUT);
@@ -2836,9 +2845,39 @@ tOffset hdfsGetUsed(hdfsFS fs)
     }
     return jVal.j;
 }
+ 
+/**
+ * We cannot add new fields to the hdfsFileInfo structure because it would break
+ * binary compatibility.  The reason is because we return an array
+ * of hdfsFileInfo structures from hdfsListDirectory.  So changing the size of
+ * those structures would break all programs that relied on finding the second
+ * element in the array at <base_offset> + sizeof(struct hdfsFileInfo).
+ *
+ * So instead, we add the new fields to the hdfsExtendedFileInfo structure.
+ * This structure is contained in the mOwner string found inside the
+ * hdfsFileInfo.  Specifically, the format of mOwner is:
+ *
+ * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure]
+ *
+ * The padding is added so that the hdfsExtendedFileInfo structure starts on an
+ * 8-byte boundary.
+ *
+ * @param str           The string to locate the extended info in.
+ * @return              The offset of the hdfsExtendedFileInfo structure.
+ */
+static size_t getExtendedFileInfoOffset(const char *str)
+{
+    int num_64_bit_words = ((strlen(str) + 1) + 7) / 8;
+    return num_64_bit_words * 8;
+}
 
+static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo)
+{
+    char *owner = fileInfo->mOwner;
+    return (struct hdfsExtendedFileInfo *)(owner +
+                getExtendedFileInfoOffset(owner));
+}
 
- 
 static jthrowable
 getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
 {
@@ -2852,6 +2891,8 @@ getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
     const char *cPathName;
     const char *cUserName;
     const char *cGroupName;
+    struct hdfsExtendedFileInfo *extInfo;
+    size_t extOffset;
 
     jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
                      HADOOP_STAT, "isDir", "()Z");
@@ -2926,9 +2967,24 @@ getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
         jthr = getPendingExceptionAndClear(env);
         goto done;
     }
-    fileInfo->mOwner = strdup(cUserName);
+    extOffset = getExtendedFileInfoOffset(cUserName);
+    fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo));
+    if (!fileInfo->mOwner) {
+        jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner");
+        goto done;
+    }
+    strcpy(fileInfo->mOwner, cUserName);
     (*env)->ReleaseStringUTFChars(env, jUserName, cUserName);
-
+    extInfo = getExtendedFileInfo(fileInfo);
+    memset(extInfo, 0, sizeof(*extInfo));
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                    HADOOP_STAT, "isEncrypted", "()Z");
+    if (jthr) {
+        goto done;
+    }
+    if (jVal.z == JNI_TRUE) {
+        extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED;
+    }
     jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
                     "getGroup", "()Ljava/lang/String;");
     if (jthr)
@@ -3174,6 +3230,13 @@ void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
     free(hdfsFileInfo);
 }
 
+int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo)
+{
+    struct hdfsExtendedFileInfo *extInfo;
+
+    extInfo = getExtendedFileInfo(fileInfo);
+    return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED);
+}
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
index 270db81..0625da3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
@@ -592,6 +592,14 @@ extern  "C" {
      */
     void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
 
+    /**
+     * hdfsFileIsEncrypted: determine if a file is encrypted based on its
+     * hdfsFileInfo.
+     * @return -1 if there was an error (errno will be set), 0 if the file is
+     *         not encrypted, 1 if the file is encrypted.
+     */
+    int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo);
+
 
     /** 
      * hdfsGetHosts - Get hostnames where a particular block (determined by

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b1c9c62/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
index cf605e3..64c1a8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
@@ -217,6 +217,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
     fileInfo = hdfsGetPathInfo(fs, paths->file2);
     EXPECT_NONNULL(fileInfo);
     EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
+    EXPECT_ZERO(hdfsFileIsEncrypted(fileInfo));
     hdfsFreeFileInfo(fileInfo, 1);
 
     EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2"));