You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/01/13 18:50:24 UTC
[23/50] [abbrv] hadoop git commit: HDFS-7589. Break the dependency
between libnative_mini_dfs and libhdfs. Contributed by Zhanwei Wang.
HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs. Contributed by Zhanwei Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/708b1aa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/708b1aa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/708b1aa0
Branch: refs/heads/HDFS-6994
Commit: 708b1aa074d98df7d8abdc805818c787ce5779b8
Parents: 20625c8
Author: cnauroth <cn...@apache.org>
Authored: Thu Jan 8 10:23:40 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Jan 8 10:23:40 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop-hdfs/src/CMakeLists.txt | 8 ++-
.../src/main/native/libhdfs/native_mini_dfs.c | 52 ++++++++++----------
.../src/main/native/libhdfs/native_mini_dfs.h | 17 +++++--
.../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 ++++++++++++
5 files changed, 80 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9f48c76..9c1be4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7561. TestFetchImage should write fetched-image-dir under target.
(Liang Xie via shv)
+ HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs.
+ (Zhanwei Wang via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 227be45..8da49c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -172,9 +172,15 @@ target_link_libraries(test_libhdfs_write
add_library(native_mini_dfs
main/native/libhdfs/native_mini_dfs.c
+ main/native/libhdfs/common/htable.c
+ main/native/libhdfs/exception.c
+ main/native/libhdfs/jni_helper.c
+ ${OS_DIR}/mutexes.c
+ ${OS_DIR}/thread_local_storage.c
)
target_link_libraries(native_mini_dfs
- hdfs
+ ${JAVA_JVM_LIBRARY}
+ ${OS_LINK_LIBRARIES}
)
add_executable(test_native_mini_dfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
index 2c42fa5..b37ebcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
@@ -17,8 +17,6 @@
*/
#include "exception.h"
-#include "hdfs.h"
-#include "hdfs_test.h"
#include "jni_helper.h"
#include "native_mini_dfs.h"
#include "platform.h"
@@ -32,6 +30,10 @@
#include <sys/types.h>
#include <unistd.h>
+#ifndef EINTERNAL
+#define EINTERNAL 255
+#endif
+
#define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
#define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
#define HADOOP_CONF "org/apache/hadoop/conf/Configuration"
@@ -52,6 +54,25 @@ struct NativeMiniDfsCluster {
char domainSocketPath[PATH_MAX];
};
+static int hdfsDisableDomainSocketSecurity(void)
+{
+ jthrowable jthr;
+ JNIEnv* env = getJNIEnv();
+ if (env == NULL) {
+ errno = EINTERNAL;
+ return -1;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
+ "org/apache/hadoop/net/unix/DomainSocket",
+ "disableBindPathValidation", "()V");
+ if (jthr) {
+ errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "DomainSocket#disableBindPathValidation");
+ return -1;
+ }
+ return 0;
+}
+
static jthrowable nmdConfigureShortCircuit(JNIEnv *env,
struct NativeMiniDfsCluster *cl, jobject cobj)
{
@@ -345,29 +366,10 @@ error_dlr_nn:
return ret;
}
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
- struct hdfsBuilder *bld)
-{
- int ret;
- tPort port;
-
- hdfsBuilderSetNameNode(bld, "localhost");
- port = (tPort)nmdGetNameNodePort(cl);
- if (port < 0) {
- fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
- return EIO;
- }
- hdfsBuilderSetNameNodePort(bld, port);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl) {
if (cl->domainSocketPath[0]) {
- ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
- if (ret) {
- return ret;
- }
- ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
- cl->domainSocketPath);
- if (ret) {
- return ret;
- }
+ return cl->domainSocketPath;
}
- return 0;
+
+ return NULL;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
index 41d69c2..ce8b1cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
@@ -21,6 +21,10 @@
#include <jni.h> /* for jboolean */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
struct hdfsBuilder;
struct NativeMiniDfsCluster;
@@ -110,13 +114,16 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
int *port, const char **hostName);
/**
- * Configure the HDFS builder appropriately to connect to this cluster.
+ * Get domain socket path set for this cluster.
*
- * @param bld The hdfs builder
+ * @param cl The cluster
*
- * @return the port, or a negative error code
+ * @return A const string of domain socket path, or NULL if not set.
*/
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
- struct hdfsBuilder *bld);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl);
+
+#ifdef __cplusplus
+}
+#endif
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
index 3774417..92941cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
@@ -201,6 +201,37 @@ static int createZeroCopyTestFile(hdfsFS fs, char *testFileName,
return 0;
}
+static int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
+ struct hdfsBuilder *bld) {
+ int ret;
+ tPort port;
+ const char *domainSocket;
+
+ hdfsBuilderSetNameNode(bld, "localhost");
+ port = (tPort) nmdGetNameNodePort(cl);
+ if (port < 0) {
+ fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
+ return EIO;
+ }
+ hdfsBuilderSetNameNodePort(bld, port);
+
+ domainSocket = hdfsGetDomainSocketPath(cl);
+
+ if (domainSocket) {
+ ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
+ if (ret) {
+ return ret;
+ }
+ ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
+ domainSocket);
+ if (ret) {
+ return ret;
+ }
+ }
+ return 0;
+}
+
+
/**
* Test that we can write a file with libhdfs and then read it back
*/