You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/10/17 05:43:45 UTC
[44/50] [abbrv] hadoop git commit: HDFS-9253. Refactor tests of
libhdfs into a directory. Contributed by Haohui Mai.
HDFS-9253. Refactor tests of libhdfs into a directory. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79b8d60d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79b8d60d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79b8d60d
Branch: refs/heads/HDFS-7966
Commit: 79b8d60d085ae196b05ff4ab511ff89f652e3c55
Parents: 52ac73f
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Oct 16 11:20:17 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Oct 16 11:20:17 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs-native-client/pom.xml | 8 +-
.../src/CMakeLists.txt | 18 +
.../src/contrib/libwebhdfs/CMakeLists.txt | 1 +
.../contrib/libwebhdfs/src/hdfs_http_client.h | 2 +-
.../contrib/libwebhdfs/src/hdfs_json_parser.c | 2 +-
.../src/contrib/libwebhdfs/src/hdfs_web.c | 4 +-
.../libwebhdfs/src/test_libwebhdfs_ops.c | 4 +-
.../libwebhdfs/src/test_libwebhdfs_read.c | 2 +-
.../libwebhdfs/src/test_libwebhdfs_threaded.c | 6 +-
.../libwebhdfs/src/test_libwebhdfs_write.c | 2 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 2 +
.../src/main/native/fuse-dfs/fuse_connect.c | 2 +-
.../main/native/fuse-dfs/fuse_context_handle.h | 2 +-
.../src/main/native/fuse-dfs/fuse_file_handle.h | 2 +-
.../src/main/native/fuse-dfs/fuse_stat_struct.h | 2 +-
.../src/main/native/fuse-dfs/fuse_trash.c | 1 -
.../src/main/native/fuse-dfs/fuse_trash.h | 2 +-
.../main/native/fuse-dfs/test/fuse_workload.c | 2 +-
.../main/native/fuse-dfs/test/test_fuse_dfs.c | 6 +-
.../main/native/libhdfs-tests/CMakeLists.txt | 41 +
.../src/main/native/libhdfs-tests/expect.c | 68 ++
.../src/main/native/libhdfs-tests/expect.h | 179 ++++
.../src/main/native/libhdfs-tests/hdfs_test.h | 64 ++
.../main/native/libhdfs-tests/native_mini_dfs.c | 357 +++++++
.../main/native/libhdfs-tests/native_mini_dfs.h | 129 +++
.../src/main/native/libhdfs-tests/test_htable.c | 100 ++
.../native/libhdfs-tests/test_libhdfs_ops.c | 540 +++++++++++
.../native/libhdfs-tests/test_libhdfs_read.c | 72 ++
.../libhdfs-tests/test_libhdfs_threaded.c | 360 +++++++
.../native/libhdfs-tests/test_libhdfs_write.c | 99 ++
.../libhdfs-tests/test_libhdfs_zerocopy.c | 280 ++++++
.../native/libhdfs-tests/test_native_mini_dfs.c | 41 +
.../src/main/native/libhdfs-tests/vecsum.c | 825 ++++++++++++++++
.../src/main/native/libhdfs/CMakeLists.txt | 95 +-
.../src/main/native/libhdfs/exception.c | 2 +-
.../src/main/native/libhdfs/expect.c | 68 --
.../src/main/native/libhdfs/expect.h | 179 ----
.../src/main/native/libhdfs/hdfs.c | 2 +-
.../src/main/native/libhdfs/hdfs.h | 939 -------------------
.../src/main/native/libhdfs/hdfs_test.h | 64 --
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 939 +++++++++++++++++++
.../src/main/native/libhdfs/native_mini_dfs.c | 357 -------
.../src/main/native/libhdfs/native_mini_dfs.h | 129 ---
.../src/main/native/libhdfs/test/test_htable.c | 100 --
.../main/native/libhdfs/test/test_libhdfs_ops.c | 540 -----------
.../native/libhdfs/test/test_libhdfs_read.c | 72 --
.../native/libhdfs/test/test_libhdfs_write.c | 99 --
.../native/libhdfs/test/test_libhdfs_zerocopy.c | 280 ------
.../src/main/native/libhdfs/test/vecsum.c | 825 ----------------
.../main/native/libhdfs/test_libhdfs_threaded.c | 360 -------
.../main/native/libhdfs/test_native_mini_dfs.c | 41 -
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
52 files changed, 4157 insertions(+), 4161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index a11370d..b8d9664 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -176,7 +176,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<echo message="Finished @{test}"/>
</sequential>
</macrodef>
- <run-test test="test_libhdfs_threaded"/>
+ <run-test test="test_libhdfs_threaded_hdfs_static"/>
<echo message="Skipping test_libhdfs_zerocopy"/>
<run-test test="test_native_mini_dfs"/>
</target>
@@ -237,9 +237,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<echo message="Finished @{test}"/>
</sequential>
</macrodef>
- <run-test test="test_libhdfs_threaded"/>
- <run-test test="test_libhdfs_zerocopy"/>
- <run-test test="test_native_mini_dfs"/>
+ <run-test test="test_libhdfs_threaded_hdfs_static"/>
+ <run-test test="test_libhdfs_zerocopy_hdfs_static"/>
+ <run-test test="../libhdfs-tests/test_native_mini_dfs"/>
</target>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index fb26bec..e3e6152 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -66,7 +66,25 @@ endif()
# Configure JNI.
include(HadoopJNI)
+function(add_libhdfs_test NAME LIBRARY)
+ set(FILES)
+ foreach(FIL ${ARGN})
+ if (IS_ABSOLUTE ${FIL})
+ list(APPEND FILES ${FIL})
+ else()
+ list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
+ endif()
+ endforeach()
+ add_executable("${NAME}_${LIBRARY}" ${FILES})
+endfunction()
+
+function(link_libhdfs_test NAME LIBRARY)
+target_link_libraries("${NAME}_${LIBRARY}" ${LIBRARY} ${ARGN})
+endfunction()
+
add_subdirectory(main/native/libhdfs)
+add_subdirectory(main/native/libhdfs-tests)
+
if(REQUIRE_LIBWEBHDFS)
add_subdirectory(contrib/libwebhdfs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/CMakeLists.txt
index 009dfd6..cc2b42d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/CMakeLists.txt
@@ -27,6 +27,7 @@ include_directories(
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/main/native
${CMAKE_SOURCE_DIR}/main/native/libhdfs
+ ${CMAKE_SOURCE_DIR}/main/native/libhdfs/include
${OS_DIR}
${JANSSON_INCLUDE_DIR}
)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_http_client.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_http_client.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_http_client.h
index 8d1c3db..ab85464 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_http_client.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_http_client.h
@@ -21,7 +21,7 @@
#ifndef _HDFS_HTTP_CLIENT_H_
#define _HDFS_HTTP_CLIENT_H_
-#include "hdfs.h" /* for tSize */
+#include "hdfs/hdfs.h" /* for tSize */
#include <pthread.h> /* for pthread_t */
#include <unistd.h> /* for size_t */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_json_parser.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_json_parser.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_json_parser.c
index 178fb9d..f0973a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_json_parser.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_json_parser.c
@@ -17,7 +17,7 @@
*/
#include "exception.h"
-#include "hdfs.h" /* for hdfsFileInfo */
+#include "hdfs/hdfs.h" /* for hdfsFileInfo */
#include "hdfs_json_parser.h"
#include <stdlib.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_web.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_web.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_web.c
index 86b4faf..a3d6575 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_web.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/hdfs_web.c
@@ -20,8 +20,8 @@
#include <string.h>
#include <stdlib.h>
-#include "exception.h"
-#include "hdfs.h"
+#include "libhdfs/exception.h"
+#include "hdfs/hdfs.h"
#include "hdfs_http_client.h"
#include "hdfs_http_query.h"
#include "hdfs_json_parser.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
index 87550ae..af748d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
@@ -16,8 +16,8 @@
* limitations under the License.
*/
-#include "hdfs.h"
-#include "native_mini_dfs.h"
+#include "hdfs/hdfs.h"
+#include "libhdfs-tests/native_mini_dfs.h"
#include <inttypes.h>
#include <jni.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
index 4bd3078..61ff113 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "hdfs.h"
+#include "hdfs/hdfs.h"
#include <stdio.h>
#include <stdlib.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
index 6c9a12e..72e333d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-#include "expect.h"
-#include "hdfs.h"
-#include "native_mini_dfs.h"
+#include "libhdfs-tests/expect.h"
+#include "hdfs/hdfs.h"
+#include "libhdfs-tests/native_mini_dfs.h"
#include <errno.h>
#include <semaphore.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
index 652fb86..2a3310a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "hdfs.h"
+#include "hdfs/hdfs.h"
#include <limits.h>
#include <stdio.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt
index 0e0db5d..44b18e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt
@@ -36,6 +36,7 @@ include_directories(
${JNI_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/main/native
${CMAKE_SOURCE_DIR}/main/native/libhdfs
+ ${CMAKE_SOURCE_DIR}/main/native/libhdfs/include
${OS_DIR}
${FUSE_INCLUDE_DIRS})
@@ -84,5 +85,6 @@ add_executable(test_fuse_dfs
target_link_libraries(test_fuse_dfs
${FUSE_LIBRARIES}
native_mini_dfs
+ ${JAVA_JVM_LIBRARY}
pthread
)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_connect.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_connect.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_connect.c
index 8a2a00b..79106bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_connect.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_connect.c
@@ -19,7 +19,7 @@
#include "fuse_connect.h"
#include "fuse_dfs.h"
#include "fuse_users.h"
-#include "libhdfs/hdfs.h"
+#include "hdfs/hdfs.h"
#include "util/tree.h"
#include <inttypes.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_context_handle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_context_handle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_context_handle.h
index 6929062..1561711 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_context_handle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_context_handle.h
@@ -19,7 +19,7 @@
#ifndef __FUSE_CONTEXT_HANDLE_H__
#define __FUSE_CONTEXT_HANDLE_H__
-#include <hdfs.h>
+#include <hdfs/hdfs.h>
#include <stddef.h>
#include <sys/types.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_file_handle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_file_handle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_file_handle.h
index 7f9346c..b04f9ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_file_handle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_file_handle.h
@@ -19,7 +19,7 @@
#ifndef __FUSE_FILE_HANDLE_H__
#define __FUSE_FILE_HANDLE_H__
-#include <hdfs.h>
+#include <hdfs/hdfs.h>
#include <pthread.h>
struct hdfsConn;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h
index d42a371..7828d54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h
@@ -23,7 +23,7 @@
#include <sys/stat.h>
#include <unistd.h>
-#include "hdfs.h"
+#include "hdfs/hdfs.h"
/**
* Converts from a hdfs hdfsFileInfo to a POSIX stat struct
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
index 5e58087..02f1b5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-#include <hdfs.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdio.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h
index 220ce3d..e0cfbad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h
@@ -19,7 +19,7 @@
#ifndef __FUSE_TRASH_H__
#define __FUSE_TRASH_H__
-#include <hdfs.h>
+#include <hdfs/hdfs.h>
int hdfsDeleteWithTrash(hdfsFS userFS, const char *path, int useTrash);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/fuse_workload.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/fuse_workload.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/fuse_workload.c
index 78fdbc6..26c482b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/fuse_workload.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/fuse_workload.c
@@ -19,7 +19,7 @@
#define FUSE_USE_VERSION 26
#include "fuse-dfs/test/fuse_workload.h"
-#include "libhdfs/expect.h"
+#include "libhdfs-tests/expect.h"
#include "util/posix_util.h"
#include <dirent.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/test_fuse_dfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/test_fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/test_fuse_dfs.c
index f4212a6..a4d8ab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/test_fuse_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/test_fuse_dfs.c
@@ -17,9 +17,9 @@
*/
#include "fuse-dfs/test/fuse_workload.h"
-#include "libhdfs/expect.h"
-#include "libhdfs/hdfs.h"
-#include "libhdfs/native_mini_dfs.h"
+#include "hdfs/hdfs.h"
+#include "libhdfs-tests/expect.h"
+#include "libhdfs-tests/native_mini_dfs.h"
#include "util/posix_util.h"
#include <ctype.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
new file mode 100644
index 0000000..473e762
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_directories(
+ ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs/include
+ ${GENERATED_JAVAH}
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs
+ ${JNI_INCLUDE_DIRS}
+ ${OS_DIR}
+)
+
+add_library(native_mini_dfs
+ native_mini_dfs.c
+ ../libhdfs/common/htable.c
+ ../libhdfs/exception.c
+ ../libhdfs/jni_helper.c
+ ${OS_DIR}/mutexes.c
+ ${OS_DIR}/thread_local_storage.c
+)
+
+add_executable(test_native_mini_dfs test_native_mini_dfs.c)
+target_link_libraries(test_native_mini_dfs native_mini_dfs ${JAVA_JVM_LIBRARY})
+
+add_executable(test_htable ../libhdfs/common/htable.c test_htable.c)
+target_link_libraries(test_htable ${OS_LINK_LIBRARIES})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.c
new file mode 100644
index 0000000..8ecfed8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.c
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "expect.h"
+#include "hdfs/hdfs.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int expectFileStats(hdfsFile file,
+ uint64_t expectedTotalBytesRead,
+ uint64_t expectedTotalLocalBytesRead,
+ uint64_t expectedTotalShortCircuitBytesRead,
+ uint64_t expectedTotalZeroCopyBytesRead)
+{
+ struct hdfsReadStatistics *stats = NULL;
+ EXPECT_ZERO(hdfsFileGetReadStatistics(file, &stats));
+ fprintf(stderr, "expectFileStats(expectedTotalBytesRead=%"PRId64", "
+ "expectedTotalLocalBytesRead=%"PRId64", "
+ "expectedTotalShortCircuitBytesRead=%"PRId64", "
+ "expectedTotalZeroCopyBytesRead=%"PRId64", "
+ "totalBytesRead=%"PRId64", "
+ "totalLocalBytesRead=%"PRId64", "
+ "totalShortCircuitBytesRead=%"PRId64", "
+ "totalZeroCopyBytesRead=%"PRId64")\n",
+ expectedTotalBytesRead,
+ expectedTotalLocalBytesRead,
+ expectedTotalShortCircuitBytesRead,
+ expectedTotalZeroCopyBytesRead,
+ stats->totalBytesRead,
+ stats->totalLocalBytesRead,
+ stats->totalShortCircuitBytesRead,
+ stats->totalZeroCopyBytesRead);
+ if (expectedTotalBytesRead != UINT64_MAX) {
+ EXPECT_UINT64_EQ(expectedTotalBytesRead, stats->totalBytesRead);
+ }
+ if (expectedTotalLocalBytesRead != UINT64_MAX) {
+ EXPECT_UINT64_EQ(expectedTotalLocalBytesRead,
+ stats->totalLocalBytesRead);
+ }
+ if (expectedTotalShortCircuitBytesRead != UINT64_MAX) {
+ EXPECT_UINT64_EQ(expectedTotalShortCircuitBytesRead,
+ stats->totalShortCircuitBytesRead);
+ }
+ if (expectedTotalZeroCopyBytesRead != UINT64_MAX) {
+ EXPECT_UINT64_EQ(expectedTotalZeroCopyBytesRead,
+ stats->totalZeroCopyBytesRead);
+ }
+ hdfsFileFreeReadStatistics(stats);
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
new file mode 100644
index 0000000..49aa285
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_NATIVE_TESTS_EXPECT_H
+#define LIBHDFS_NATIVE_TESTS_EXPECT_H
+
+#include <inttypes.h>
+#include <stdio.h>
+
+struct hdfsFile_internal;
+
+#define EXPECT_ZERO(x) \
+ do { \
+ int __my_ret__ = x; \
+ if (__my_ret__) { \
+ int __my_errno__ = errno; \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): got nonzero from %s\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, #x); \
+ return __my_ret__; \
+ } \
+ } while (0);
+
+#define EXPECT_NULL(x) \
+ do { \
+ const void* __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != NULL) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got non-NULL value %p from %s\n", \
+ __FILE__, __LINE__, __my_errno__, __my_ret__, #x); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_NULL_WITH_ERRNO(x, e) \
+ do { \
+ const void* __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != NULL) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got non-NULL value %p from %s\n", \
+ __FILE__, __LINE__, __my_errno__, __my_ret__, #x); \
+ return -1; \
+ } \
+ if (__my_errno__ != e) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got expected NULL without expected errno %d from %s\n", \
+ __FILE__, __LINE__, __my_errno__, e, #x); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_NONNULL(x) \
+ do { \
+ const void* __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ == NULL) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got NULL from %s\n", __FILE__, __LINE__, __my_errno__, #x); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_NEGATIVE_ONE_WITH_ERRNO(x, e) \
+ do { \
+ int __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != -1) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): expected -1 from %s\n", \
+ __FILE__, __LINE__, \
+ __my_ret__, __my_errno__, #x); \
+ return -1; \
+ } \
+ if (__my_errno__ != e) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): expected errno = %d from %s\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, e, #x); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_NONZERO(x) \
+ do { \
+ int __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (!__my_ret__) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): got zero from %s\n", __FILE__, __LINE__, \
+ __my_ret__, __my_errno__, #x); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_NONNEGATIVE(x) \
+ do { \
+ int __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ < 0) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): got negative return from %s\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, #x); \
+ return __my_ret__; \
+ } \
+ } while (0);
+
+#define EXPECT_INT_EQ(x, y) \
+ do { \
+ int __my_ret__ = y; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != (x)) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "code %d (errno: %d): expected %d\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, (x)); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_INT64_EQ(x, y) \
+ do { \
+ int64_t __my_ret__ = y; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != (x)) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "value %"PRId64" (errno: %d): expected %"PRId64"\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, (x)); \
+ return -1; \
+ } \
+ } while (0);
+
+#define EXPECT_UINT64_EQ(x, y) \
+ do { \
+ uint64_t __my_ret__ = y; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != (x)) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "value %"PRIu64" (errno: %d): expected %"PRIu64"\n", \
+ __FILE__, __LINE__, __my_ret__, __my_errno__, (x)); \
+ return -1; \
+ } \
+ } while (0);
+
+#define RETRY_ON_EINTR_GET_ERRNO(ret, expr) do { \
+ ret = expr; \
+ if (!ret) \
+ break; \
+ ret = -errno; \
+ } while (ret == -EINTR);
+
+/**
+ * Test that an HDFS file has the given statistics.
+ *
+ * Any parameter can be set to UINT64_MAX to avoid checking it.
+ *
+ * @return 0 on success; error code otherwise
+ */
+int expectFileStats(struct hdfsFile_internal *file,
+ uint64_t expectedTotalBytesRead,
+ uint64_t expectedTotalLocalBytesRead,
+ uint64_t expectedTotalShortCircuitBytesRead,
+ uint64_t expectedTotalZeroCopyBytesRead);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/hdfs_test.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/hdfs_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/hdfs_test.h
new file mode 100644
index 0000000..0eab9a6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/hdfs_test.h
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_HDFS_TEST_H
+#define LIBHDFS_HDFS_TEST_H
+
+struct hdfsFile_internal;
+
+/**
+ * Some functions that are visible only for testing.
+ *
+ * This header is not meant to be exported or used outside of the libhdfs unit
+ * tests.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ /**
+ * Determine if a file is using the "direct read" optimization.
+ *
+ * @param file The HDFS file
+ * @return 1 if the file is using the direct read optimization,
+ * 0 otherwise.
+ */
+ int hdfsFileUsesDirectRead(struct hdfsFile_internal *file);
+
+ /**
+ * Disable the direct read optimization for a file.
+ *
+ * This is mainly provided for unit testing purposes.
+ *
+ * @param file The HDFS file
+ */
+ void hdfsFileDisableDirectRead(struct hdfsFile_internal *file);
+
+ /**
+ * Disable domain socket security checks.
+ *
+ * @param 0 if domain socket security was disabled;
+ * -1 if not.
+ */
+ int hdfsDisableDomainSocketSecurity(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
new file mode 100644
index 0000000..b36ef76
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "exception.h"
+#include "jni_helper.h"
+#include "native_mini_dfs.h"
+#include "platform.h"
+
+#include <errno.h>
+#include <jni.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#ifndef EINTERNAL
+#define EINTERNAL 255
+#endif
+
+#define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
+#define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
+#define HADOOP_CONF "org/apache/hadoop/conf/Configuration"
+#define HADOOP_NAMENODE "org/apache/hadoop/hdfs/server/namenode/NameNode"
+#define JAVA_INETSOCKETADDRESS "java/net/InetSocketAddress"
+
+struct NativeMiniDfsCluster {
+ /**
+ * The NativeMiniDfsCluster object
+ */
+ jobject obj;
+
+ /**
+ * Path to the domain socket, or the empty string if there is none.
+ */
+ char domainSocketPath[PATH_MAX];
+};
+
+static int hdfsDisableDomainSocketSecurity(void)
+{
+ jthrowable jthr;
+ JNIEnv* env = getJNIEnv();
+ if (env == NULL) {
+ errno = EINTERNAL;
+ return -1;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
+ "org/apache/hadoop/net/unix/DomainSocket",
+ "disableBindPathValidation", "()V");
+ if (jthr) {
+ errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "DomainSocket#disableBindPathValidation");
+ return -1;
+ }
+ return 0;
+}
+
+static jthrowable nmdConfigureShortCircuit(JNIEnv *env,
+ struct NativeMiniDfsCluster *cl, jobject cobj)
+{
+ jthrowable jthr;
+ char *tmpDir;
+
+ int ret = hdfsDisableDomainSocketSecurity();
+ if (ret) {
+ return newRuntimeError(env, "failed to disable hdfs domain "
+ "socket security: error %d", ret);
+ }
+ jthr = hadoopConfSetStr(env, cobj, "dfs.client.read.shortcircuit", "true");
+ if (jthr) {
+ return jthr;
+ }
+ tmpDir = getenv("TMPDIR");
+ if (!tmpDir) {
+ tmpDir = "/tmp";
+ }
+ snprintf(cl->domainSocketPath, PATH_MAX, "%s/native_mini_dfs.sock.%d.%d",
+ tmpDir, getpid(), rand());
+ snprintf(cl->domainSocketPath, PATH_MAX, "%s/native_mini_dfs.sock.%d.%d",
+ tmpDir, getpid(), rand());
+ jthr = hadoopConfSetStr(env, cobj, "dfs.domain.socket.path",
+ cl->domainSocketPath);
+ if (jthr) {
+ return jthr;
+ }
+ return NULL;
+}
+
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
+{
+ struct NativeMiniDfsCluster* cl = NULL;
+ jobject bld = NULL, cobj = NULL, cluster = NULL;
+ jvalue val;
+ JNIEnv *env = getJNIEnv();
+ jthrowable jthr;
+ jstring jconfStr = NULL;
+
+ if (!env) {
+ fprintf(stderr, "nmdCreate: unable to construct JNIEnv.\n");
+ return NULL;
+ }
+ cl = calloc(1, sizeof(struct NativeMiniDfsCluster));
+ if (!cl) {
+ fprintf(stderr, "nmdCreate: OOM");
+ goto error;
+ }
+ jthr = constructNewObjectOfClass(env, &cobj, HADOOP_CONF, "()V");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: new Configuration");
+ goto error;
+ }
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: Configuration::setBoolean");
+ goto error;
+ }
+ // Disable 'minimum block size' -- it's annoying in tests.
+ (*env)->DeleteLocalRef(env, jconfStr);
+ jconfStr = NULL;
+ jthr = newJavaStr(env, "dfs.namenode.fs-limits.min-block-size", &jconfStr);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: new String");
+ goto error;
+ }
+ jthr = invokeMethod(env, NULL, INSTANCE, cobj, HADOOP_CONF,
+ "setLong", "(Ljava/lang/String;J)V", jconfStr, 0LL);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: Configuration::setLong");
+ goto error;
+ }
+ // Creae MiniDFSCluster object
+ jthr = constructNewObjectOfClass(env, &bld, MINIDFS_CLUSTER_BUILDER,
+ "(L"HADOOP_CONF";)V", cobj);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: NativeMiniDfsCluster#Builder#Builder");
+ goto error;
+ }
+ if (conf->configureShortCircuit) {
+ jthr = nmdConfigureShortCircuit(env, cl, cobj);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: nmdConfigureShortCircuit error");
+ goto error;
+ }
+ }
+ jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+ "format", "(Z)L" MINIDFS_CLUSTER_BUILDER ";", conf->doFormat);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
+ "Builder::format");
+ goto error;
+ }
+ (*env)->DeleteLocalRef(env, val.l);
+ if (conf->webhdfsEnabled) {
+ jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+ "nameNodeHttpPort", "(I)L" MINIDFS_CLUSTER_BUILDER ";",
+ conf->namenodeHttpPort);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
+ "Builder::nameNodeHttpPort");
+ goto error;
+ }
+ (*env)->DeleteLocalRef(env, val.l);
+ }
+ jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+ "build", "()L" MINIDFS_CLUSTER ";");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdCreate: Builder#build");
+ goto error;
+ }
+ cluster = val.l;
+ cl->obj = (*env)->NewGlobalRef(env, val.l);
+ if (!cl->obj) {
+ printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+ "nmdCreate: NewGlobalRef");
+ goto error;
+ }
+ (*env)->DeleteLocalRef(env, cluster);
+ (*env)->DeleteLocalRef(env, bld);
+ (*env)->DeleteLocalRef(env, cobj);
+ (*env)->DeleteLocalRef(env, jconfStr);
+ return cl;
+
+error:
+ (*env)->DeleteLocalRef(env, cluster);
+ (*env)->DeleteLocalRef(env, bld);
+ (*env)->DeleteLocalRef(env, cobj);
+ (*env)->DeleteLocalRef(env, jconfStr);
+ free(cl);
+ return NULL;
+}
+
+void nmdFree(struct NativeMiniDfsCluster* cl)
+{
+ JNIEnv *env = getJNIEnv();
+ if (!env) {
+ fprintf(stderr, "nmdFree: getJNIEnv failed\n");
+ free(cl);
+ return;
+ }
+ (*env)->DeleteGlobalRef(env, cl->obj);
+ free(cl);
+}
+
+int nmdShutdown(struct NativeMiniDfsCluster* cl)
+{
+ JNIEnv *env = getJNIEnv();
+ jthrowable jthr;
+
+ if (!env) {
+ fprintf(stderr, "nmdShutdown: getJNIEnv failed\n");
+ return -EIO;
+ }
+ jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
+ MINIDFS_CLUSTER, "shutdown", "()V");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdShutdown: MiniDFSCluster#shutdown");
+ return -EIO;
+ }
+ return 0;
+}
+
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl)
+{
+ jthrowable jthr;
+ JNIEnv *env = getJNIEnv();
+ if (!env) {
+ fprintf(stderr, "nmdWaitClusterUp: getJNIEnv failed\n");
+ return -EIO;
+ }
+ jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
+ MINIDFS_CLUSTER, "waitClusterUp", "()V");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdWaitClusterUp: MiniDFSCluster#waitClusterUp ");
+ return -EIO;
+ }
+ return 0;
+}
+
+int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl)
+{
+ JNIEnv *env = getJNIEnv();
+ jvalue jVal;
+ jthrowable jthr;
+
+ if (!env) {
+ fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
+ return -EIO;
+ }
+ // Note: this will have to be updated when HA nativeMiniDfs clusters are
+ // supported
+ jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj,
+ MINIDFS_CLUSTER, "getNameNodePort", "()I");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdHdfsConnect: MiniDFSCluster#getNameNodePort");
+ return -EIO;
+ }
+ return jVal.i;
+}
+
+int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
+ int *port, const char **hostName)
+{
+ JNIEnv *env = getJNIEnv();
+ jvalue jVal;
+ jobject jNameNode, jAddress;
+ jthrowable jthr;
+ int ret = 0;
+ const char *host;
+
+ if (!env) {
+ fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
+ return -EIO;
+ }
+ // First get the (first) NameNode of the cluster
+ jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj, MINIDFS_CLUSTER,
+ "getNameNode", "()L" HADOOP_NAMENODE ";");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdGetNameNodeHttpAddress: "
+ "MiniDFSCluster#getNameNode");
+ return -EIO;
+ }
+ jNameNode = jVal.l;
+
+ // Then get the http address (InetSocketAddress) of the NameNode
+ jthr = invokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE,
+ "getHttpAddress", "()L" JAVA_INETSOCKETADDRESS ";");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdGetNameNodeHttpAddress: "
+ "NameNode#getHttpAddress");
+ goto error_dlr_nn;
+ }
+ jAddress = jVal.l;
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, jAddress,
+ JAVA_INETSOCKETADDRESS, "getPort", "()I");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdGetNameNodeHttpAddress: "
+ "InetSocketAddress#getPort");
+ goto error_dlr_addr;
+ }
+ *port = jVal.i;
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS,
+ "getHostName", "()Ljava/lang/String;");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "nmdGetNameNodeHttpAddress: "
+ "InetSocketAddress#getHostName");
+ goto error_dlr_addr;
+ }
+ host = (*env)->GetStringUTFChars(env, jVal.l, NULL);
+ *hostName = strdup(host);
+ (*env)->ReleaseStringUTFChars(env, jVal.l, host);
+
+error_dlr_addr:
+ (*env)->DeleteLocalRef(env, jAddress);
+error_dlr_nn:
+ (*env)->DeleteLocalRef(env, jNameNode);
+
+ return ret;
+}
+
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl) {
+ if (cl->domainSocketPath[0]) {
+ return cl->domainSocketPath;
+ }
+
+ return NULL;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
new file mode 100644
index 0000000..ce8b1cf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_NATIVE_MINI_DFS_H
+#define LIBHDFS_NATIVE_MINI_DFS_H
+
+#include <jni.h> /* for jboolean */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct hdfsBuilder;
+struct NativeMiniDfsCluster;
+
+/**
+ * Represents a configuration to use for creating a Native MiniDFSCluster
+ */
+struct NativeMiniDfsConf {
+ /**
+ * Nonzero if the cluster should be formatted prior to startup.
+ */
+ jboolean doFormat;
+
+ /**
+ * Whether or not to enable webhdfs in MiniDfsCluster
+ */
+ jboolean webhdfsEnabled;
+
+ /**
+ * The http port of the namenode in MiniDfsCluster
+ */
+ jint namenodeHttpPort;
+
+ /**
+ * Nonzero if we should configure short circuit.
+ */
+ jboolean configureShortCircuit;
+};
+
+/**
+ * Create a NativeMiniDfsBuilder
+ *
+ * @param conf (inout) The cluster configuration
+ *
+ * @return a NativeMiniDfsBuilder, or a NULL pointer on error.
+ */
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf);
+
+/**
+ * Wait until a MiniDFSCluster comes out of safe mode.
+ *
+ * @param cl The cluster
+ *
+ * @return 0 on success; a non-zero error code if the cluster fails to
+ * come out of safe mode.
+ */
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Shut down a NativeMiniDFS cluster
+ *
+ * @param cl The cluster
+ *
+ * @return 0 on success; a non-zero error code if an exception is
+ * thrown.
+ */
+int nmdShutdown(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Destroy a Native MiniDFSCluster
+ *
+ * @param cl The cluster to destroy
+ */
+void nmdFree(struct NativeMiniDfsCluster* cl);
+
+/**
+ * Get the port that's in use by the given (non-HA) nativeMiniDfs
+ *
+ * @param cl The initialized NativeMiniDfsCluster
+ *
+ * @return the port, or a negative error code
+ */
+int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl);
+
+/**
+ * Get the http address that's in use by the given (non-HA) nativeMiniDfs
+ *
+ * @param cl The initialized NativeMiniDfsCluster
+ * @param port Used to capture the http port of the NameNode
+ * of the NativeMiniDfsCluster
+ * @param hostName Used to capture the http hostname of the NameNode
+ * of the NativeMiniDfsCluster
+ *
+ * @return 0 on success; a non-zero error code if failing to
+ * get the information.
+ */
+int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
+ int *port, const char **hostName);
+
+/**
+ * Get domain socket path set for this cluster.
+ *
+ * @param cl The cluster
+ *
+ * @return A const string of domain socket path, or NULL if not set.
+ */
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_htable.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_htable.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_htable.c
new file mode 100644
index 0000000..0c3861b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_htable.c
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/htable.h"
+#include "expect.h"
+#include "hdfs_test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Disable type cast and loss of precision warnings, because the test
+// manipulates void* values manually on purpose.
+#ifdef WIN32
+#pragma warning(disable: 4244 4306)
+#endif
+
+static uint32_t simple_hash(const void *key, uint32_t size)
+{
+ uintptr_t k = (uintptr_t)key;
+ return ((13 + k) * 6367) % size;
+}
+
+static int simple_compare(const void *a, const void *b)
+{
+ return a == b;
+}
+
+static void expect_102(void *f, void *k, void *v)
+{
+ int *found_102 = f;
+ uintptr_t key = (uintptr_t)k;
+ uintptr_t val = (uintptr_t)v;
+
+ if ((key == 2) && (val == 102)) {
+ *found_102 = 1;
+ } else {
+ abort();
+ }
+}
+
+static void *htable_pop_val(struct htable *ht, void *key)
+{
+ void *old_key, *old_val;
+
+ htable_pop(ht, key, &old_key, &old_val);
+ return old_val;
+}
+
+int main(void)
+{
+ struct htable *ht;
+ int found_102 = 0;
+
+ ht = htable_alloc(4, simple_hash, simple_compare);
+ EXPECT_INT_EQ(0, htable_used(ht));
+ EXPECT_INT_EQ(4, htable_capacity(ht));
+ EXPECT_NULL(htable_get(ht, (void*)123));
+ EXPECT_NULL(htable_pop_val(ht, (void*)123));
+ EXPECT_ZERO(htable_put(ht, (void*)123, (void*)456));
+ EXPECT_INT_EQ(456, (uintptr_t)htable_get(ht, (void*)123));
+ EXPECT_INT_EQ(456, (uintptr_t)htable_pop_val(ht, (void*)123));
+ EXPECT_NULL(htable_pop_val(ht, (void*)123));
+
+ // Enlarge the hash table
+ EXPECT_ZERO(htable_put(ht, (void*)1, (void*)101));
+ EXPECT_ZERO(htable_put(ht, (void*)2, (void*)102));
+ EXPECT_ZERO(htable_put(ht, (void*)3, (void*)103));
+ EXPECT_INT_EQ(3, htable_used(ht));
+ EXPECT_INT_EQ(8, htable_capacity(ht));
+ EXPECT_INT_EQ(102, (uintptr_t)htable_get(ht, (void*)2));
+ EXPECT_INT_EQ(101, (uintptr_t)htable_pop_val(ht, (void*)1));
+ EXPECT_INT_EQ(103, (uintptr_t)htable_pop_val(ht, (void*)3));
+ EXPECT_INT_EQ(1, htable_used(ht));
+ htable_visit(ht, expect_102, &found_102);
+ EXPECT_INT_EQ(1, found_102);
+ htable_free(ht);
+
+ fprintf(stderr, "SUCCESS.\n");
+ return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
new file mode 100644
index 0000000..d69aa37
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
@@ -0,0 +1,540 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs/hdfs.h"
+#include "hdfs_test.h"
+#include "platform.h"
+
+#include <inttypes.h>
+#include <jni.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+void permission_disp(short permissions, char *rtr) {
+ int i;
+ short permissionsId;
+ char* perm;
+ rtr[9] = '\0';
+ for(i=2;i>=0;i--)
+ {
+ permissionsId = permissions >> (i * 3) & (short)7;
+ switch(permissionsId) {
+ case 7:
+ perm = "rwx"; break;
+ case 6:
+ perm = "rw-"; break;
+ case 5:
+ perm = "r-x"; break;
+ case 4:
+ perm = "r--"; break;
+ case 3:
+ perm = "-wx"; break;
+ case 2:
+ perm = "-w-"; break;
+ case 1:
+ perm = "--x"; break;
+ case 0:
+ perm = "---"; break;
+ default:
+ perm = "???";
+ }
+ strncpy(rtr, perm, 3);
+ rtr+=3;
+ }
+}
+
+int main(int argc, char **argv) {
+ const char *writePath = "/tmp/testfile.txt";
+ const char *fileContents = "Hello, World!";
+ const char *readPath = "/tmp/testfile.txt";
+ const char *srcPath = "/tmp/testfile.txt";
+ const char *dstPath = "/tmp/testfile2.txt";
+ const char *slashTmp = "/tmp";
+ const char *newDirectory = "/tmp/newdir";
+ const char *newOwner = "root";
+ const char *tuser = "nobody";
+ const char *appendPath = "/tmp/appends";
+ const char *userPath = "/tmp/usertestfile.txt";
+
+ char buffer[32], buffer2[256], rdbuffer[32];
+ tSize num_written_bytes, num_read_bytes;
+ hdfsFS fs, lfs;
+ hdfsFile writeFile, readFile, localFile, appendFile, userFile;
+ tOffset currentPos, seekPos;
+ int exists, totalResult, result, numEntries, i, j;
+ const char *resp;
+ hdfsFileInfo *fileInfo, *fileList, *finfo;
+ char *buffer3;
+ char permissions[10];
+ char ***hosts;
+ short newPerm = 0666;
+ tTime newMtime, newAtime;
+
+ fs = hdfsConnectNewInstance("default", 0);
+ if(!fs) {
+ fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
+ exit(-1);
+ }
+
+ lfs = hdfsConnectNewInstance(NULL, 0);
+ if(!lfs) {
+ fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
+ exit(-1);
+ }
+
+ {
+ //Write tests
+
+ writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
+ if(!writeFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", writePath);
+ exit(-1);
+ }
+ fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
+ num_written_bytes =
+ hdfsWrite(fs, writeFile, (void*)fileContents,
+ (tSize)(strlen(fileContents)+1));
+ if (num_written_bytes != strlen(fileContents) + 1) {
+ fprintf(stderr, "Failed to write correct number of bytes - expected %d, got %d\n",
+ (int)(strlen(fileContents) + 1), (int)num_written_bytes);
+ exit(-1);
+ }
+ fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+ currentPos = -1;
+ if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
+ fprintf(stderr,
+ "Failed to get current file position correctly! Got %" PRId64 "!\n",
+ currentPos);
+ exit(-1);
+ }
+ fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
+
+ if (hdfsFlush(fs, writeFile)) {
+ fprintf(stderr, "Failed to 'flush' %s\n", writePath);
+ exit(-1);
+ }
+ fprintf(stderr, "Flushed %s successfully!\n", writePath);
+
+ if (hdfsHFlush(fs, writeFile)) {
+ fprintf(stderr, "Failed to 'hflush' %s\n", writePath);
+ exit(-1);
+ }
+ fprintf(stderr, "HFlushed %s successfully!\n", writePath);
+
+ hdfsCloseFile(fs, writeFile);
+ }
+
+ {
+ //Read tests
+
+ exists = hdfsExists(fs, readPath);
+
+ if (exists) {
+ fprintf(stderr, "Failed to validate existence of %s\n", readPath);
+ exit(-1);
+ }
+
+ readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
+ if (!readFile) {
+ fprintf(stderr, "Failed to open %s for reading!\n", readPath);
+ exit(-1);
+ }
+
+ if (!hdfsFileIsOpenForRead(readFile)) {
+ fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file "
+ "with O_RDONLY, and it did not show up as 'open for "
+ "read'\n");
+ exit(-1);
+ }
+
+ fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
+
+ seekPos = 1;
+ if(hdfsSeek(fs, readFile, seekPos)) {
+ fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
+ exit(-1);
+ }
+
+ currentPos = -1;
+ if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
+ fprintf(stderr,
+ "Failed to get current file position correctly! Got %" PRId64 "!\n",
+ currentPos);
+ exit(-1);
+ }
+ fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
+
+ if (!hdfsFileUsesDirectRead(readFile)) {
+ fprintf(stderr, "Direct read support incorrectly not detected "
+ "for HDFS filesystem\n");
+ exit(-1);
+ }
+
+ fprintf(stderr, "Direct read support detected for HDFS\n");
+
+ // Test the direct read path
+ if(hdfsSeek(fs, readFile, 0)) {
+ fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
+ exit(-1);
+ }
+ memset(buffer, 0, sizeof(buffer));
+ num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
+ sizeof(buffer));
+ if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
+ fprintf(stderr, "Failed to read (direct). Expected %s but got %s (%d bytes)\n",
+ fileContents, buffer, num_read_bytes);
+ exit(-1);
+ }
+ fprintf(stderr, "Read (direct) following %d bytes:\n%s\n",
+ num_read_bytes, buffer);
+ if (hdfsSeek(fs, readFile, 0L)) {
+ fprintf(stderr, "Failed to seek to file start!\n");
+ exit(-1);
+ }
+
+ // Disable the direct read path so that we really go through the slow
+ // read path
+ hdfsFileDisableDirectRead(readFile);
+
+ num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
+ sizeof(buffer));
+ fprintf(stderr, "Read following %d bytes:\n%s\n",
+ num_read_bytes, buffer);
+
+ memset(buffer, 0, strlen(fileContents + 1));
+
+ num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
+ sizeof(buffer));
+ fprintf(stderr, "Read following %d bytes:\n%s\n",
+ num_read_bytes, buffer);
+
+ hdfsCloseFile(fs, readFile);
+
+ // Test correct behaviour for unsupported filesystems
+ localFile = hdfsOpenFile(lfs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
+ if(!localFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", writePath);
+ exit(-1);
+ }
+
+ num_written_bytes = hdfsWrite(lfs, localFile, (void*)fileContents,
+ (tSize)(strlen(fileContents) + 1));
+
+ hdfsCloseFile(lfs, localFile);
+ localFile = hdfsOpenFile(lfs, writePath, O_RDONLY, 0, 0, 0);
+
+ if (hdfsFileUsesDirectRead(localFile)) {
+ fprintf(stderr, "Direct read support incorrectly detected for local "
+ "filesystem\n");
+ exit(-1);
+ }
+
+ hdfsCloseFile(lfs, localFile);
+ }
+
+ totalResult = 0;
+ result = 0;
+ {
+ //Generic file-system operations
+
+ fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
+ totalResult += (resp ? 0 : 1);
+ fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
+ totalResult += (resp ? 0 : 1);
+
+ fprintf(stderr, "hdfsGetDefaultBlockSize: %" PRId64 "\n", hdfsGetDefaultBlockSize(fs));
+ fprintf(stderr, "hdfsGetCapacity: %" PRId64 "\n", hdfsGetCapacity(fs));
+ fprintf(stderr, "hdfsGetUsed: %" PRId64 "\n", hdfsGetUsed(fs));
+
+ fileInfo = NULL;
+ if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
+ fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
+ fprintf(stderr, "Name: %s, ", fileInfo->mName);
+ fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
+ fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
+ fprintf(stderr, "BlockSize: %" PRId64 ", ", fileInfo->mBlockSize);
+ fprintf(stderr, "Size: %" PRId64 ", ", fileInfo->mSize);
+ fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
+ fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
+ fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
+ permission_disp(fileInfo->mPermissions, permissions);
+ fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
+ hdfsFreeFileInfo(fileInfo, 1);
+ } else {
+ totalResult++;
+ fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
+ }
+
+ fileList = 0;
+ fileList = hdfsListDirectory(fs, newDirectory, &numEntries);
+ if (!(fileList == NULL && numEntries == 0 && !errno)) {
+ fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory);
+ totalResult++;
+ } else {
+ fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory);
+ }
+
+ fileList = 0;
+ if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
+ for(i=0; i < numEntries; ++i) {
+ fprintf(stderr, "Name: %s, ", fileList[i].mName);
+ fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
+ fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
+ fprintf(stderr, "BlockSize: %" PRId64 ", ", fileList[i].mBlockSize);
+ fprintf(stderr, "Size: %" PRId64 ", ", fileList[i].mSize);
+ fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
+ fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
+ fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
+ permission_disp(fileList[i].mPermissions, permissions);
+ fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
+ }
+ hdfsFreeFileInfo(fileList, numEntries);
+ } else {
+ if (errno) {
+ totalResult++;
+ fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
+ } else {
+ fprintf(stderr, "Empty directory!\n");
+ }
+ }
+
+ hosts = hdfsGetHosts(fs, srcPath, 0, 1);
+ if(hosts) {
+ fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
+ i=0;
+ while(hosts[i]) {
+ j = 0;
+ while(hosts[i][j]) {
+ fprintf(stderr,
+ "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
+ ++j;
+ }
+ ++i;
+ }
+ } else {
+ totalResult++;
+ fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
+ }
+
+ // setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
+
+ // chown write
+ fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ // chmod write
+ fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+
+
+ sleep(2);
+ newMtime = time(NULL);
+ newAtime = time(NULL);
+
+ // utime write
+ fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) != 0 ? "Failed!" : "Success!"));
+
+ totalResult += result;
+
+ // chown/chmod/utime read
+ finfo = hdfsGetPathInfo(fs, writePath);
+
+ fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner))) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ // will later use /tmp/ as a different user so enable it
+ fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ fprintf(stderr,"newMTime=%ld\n",newMtime);
+ fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
+
+
+ fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+
+ // No easy way to turn on access times from hdfs_test right now
+ // fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) != 0 ? "Failed!" : "Success!"));
+ // totalResult += result;
+
+ hdfsFreeFileInfo(finfo, 1);
+
+ // Clean up
+ fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory, 1)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath, 1)) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) != 0 ? "Success!" : "Failed!"));
+ totalResult += (result ? 0 : 1);
+ }
+
+ {
+ // TEST APPENDS
+
+ // CREATE
+ appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0);
+ if(!appendFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
+
+ buffer3 = "Hello,";
+ num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
+ (tSize)strlen(buffer3));
+ fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+ if (hdfsFlush(fs, appendFile)) {
+ fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Flushed %s successfully!\n", appendPath);
+
+ hdfsCloseFile(fs, appendFile);
+
+ // RE-OPEN
+ appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY|O_APPEND, 0, 0, 0);
+ if(!appendFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
+
+ buffer3 = " World";
+ num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
+ (tSize)(strlen(buffer3) + 1));
+ fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+ if (hdfsFlush(fs, appendFile)) {
+ fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Flushed %s successfully!\n", appendPath);
+
+ hdfsCloseFile(fs, appendFile);
+
+ // CHECK size
+ finfo = hdfsGetPathInfo(fs, appendPath);
+ fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize == (tOffset)(strlen("Hello, World") + 1))) == 1 ? "Success!" : "Failed!"));
+ totalResult += (result ? 0 : 1);
+
+ // READ and check data
+ readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0);
+ if (!readFile) {
+ fprintf(stderr, "Failed to open %s for reading!\n", appendPath);
+ exit(-1);
+ }
+
+ num_read_bytes = hdfsRead(fs, readFile, (void*)rdbuffer, sizeof(rdbuffer));
+ fprintf(stderr, "Read following %d bytes:\n%s\n",
+ num_read_bytes, rdbuffer);
+
+ fprintf(stderr, "read == Hello, World %s\n", ((result = (strcmp(rdbuffer, "Hello, World"))) == 0 ? "Success!" : "Failed!"));
+
+ hdfsCloseFile(fs, readFile);
+
+ // DONE test appends
+ }
+
+
+ totalResult += (hdfsDisconnect(fs) != 0);
+
+ {
+ //
+ // Now test as connecting as a specific user
+ // This is only meant to test that we connected as that user, not to test
+ // the actual fs user capabilities. Thus just create a file and read
+ // the owner is correct.
+
+ fs = hdfsConnectAsUserNewInstance("default", 0, tuser);
+ if(!fs) {
+ fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
+ exit(-1);
+ }
+
+ userFile = hdfsOpenFile(fs, userPath, O_WRONLY|O_CREAT, 0, 0, 0);
+ if(!userFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", userPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Opened %s for writing successfully...\n", userPath);
+
+ num_written_bytes = hdfsWrite(fs, userFile, (void*)fileContents,
+ (tSize)(strlen(fileContents)+1));
+ fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+ if (hdfsFlush(fs, userFile)) {
+ fprintf(stderr, "Failed to 'flush' %s\n", userPath);
+ exit(-1);
+ }
+ fprintf(stderr, "Flushed %s successfully!\n", userPath);
+
+ hdfsCloseFile(fs, userFile);
+
+ finfo = hdfsGetPathInfo(fs, userPath);
+ fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser))) != 0 ? "Failed!" : "Success!"));
+ totalResult += result;
+ }
+
+ totalResult += (hdfsDisconnect(fs) != 0);
+
+ if (totalResult != 0) {
+ return -1;
+ } else {
+ return 0;
+ }
+}
+
+/**
+ * vim: ts=4: sw=4: et:
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d60d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_read.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_read.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_read.c
new file mode 100644
index 0000000..4b90f2a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_read.c
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs/hdfs.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+ hdfsFS fs;
+ const char *rfile = argv[1];
+ tSize bufferSize = strtoul(argv[3], NULL, 10);
+ hdfsFile readFile;
+ char* buffer;
+ tSize curSize;
+
+ if (argc != 4) {
+ fprintf(stderr, "Usage: hdfs_read <filename> <filesize> <buffersize>\n");
+ exit(-1);
+ }
+
+ fs = hdfsConnect("default", 0);
+ if (!fs) {
+ fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
+ exit(-1);
+ }
+
+ readFile = hdfsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0);
+ if (!readFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", rfile);
+ exit(-2);
+ }
+
+ // data to be written to the file
+ buffer = malloc(sizeof(char) * bufferSize);
+ if(buffer == NULL) {
+ return -2;
+ }
+
+ // read from the file
+ curSize = bufferSize;
+ for (; curSize == bufferSize;) {
+ curSize = hdfsRead(fs, readFile, (void*)buffer, curSize);
+ }
+
+
+ free(buffer);
+ hdfsCloseFile(fs, readFile);
+ hdfsDisconnect(fs);
+
+ return 0;
+}
+
+/**
+ * vim: ts=4: sw=4: et:
+ */
+