You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/13 05:54:07 UTC
[01/50] [abbrv] git commit: MAPREDUCE-5985. native-task: Fix build on
macosx. Contributed by Binglin Chang
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 7aeda47bd -> ebdd5e50d
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1612659 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e788e8b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e788e8b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e788e8b6
Branch: refs/heads/HDFS-6581
Commit: e788e8b6eeafc1d20b5d51f5e2374610bc37304b
Parents: b2551c0
Author: Todd Lipcon <to...@apache.org>
Authored: Tue Jul 22 19:55:03 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Jul 22 19:55:03 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 4 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 77 ++++++------
.../src/CMakeLists.txt | 126 +++++++++----------
.../src/main/native/src/lib/commons.h | 1 +
.../src/main/native/test/TestIFile.cc | 5 +-
5 files changed, 111 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e788e8b6/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
new file mode 100644
index 0000000..18bf029
--- /dev/null
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -0,0 +1,4 @@
+Changes for Hadoop Native Map Output Collector
+------------------------------------------------
+
+MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e788e8b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 6f46fd5..40b6520 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -12,7 +12,8 @@
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
@@ -29,7 +30,7 @@
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
- </properties>
+ </properties>
<dependencies>
@@ -53,7 +54,7 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
+ <artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -85,7 +86,7 @@
<include>**/compresstest/*Test.java</include>
<include>**/nonsorttest/*Test.java</include>
<include>**/kvtest/*Test.java</include>
- </includes>
+ </includes>
<additionalClasspathElements>
<additionalClasspathElement>${basedir}/../hadoop-mapreduce-client-common/target/classes</additionalClasspathElement>
</additionalClasspathElements>
@@ -131,26 +132,26 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>native-maven-plugin</artifactId>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>javah</goal>
- </goals>
- <configuration>
- <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
- <javahClassNames>
- <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName>
- <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName>
- </javahClassNames>
- <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>native-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>javah</goal>
+ </goals>
+ <configuration>
+ <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+ <javahClassNames>
+ <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName>
+ <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName>
+ </javahClassNames>
+ <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
@@ -158,23 +159,27 @@
<execution>
<id>make</id>
<phase>compile</phase>
- <goals><goal>run</goal></goals>
+ <goals>
+ <goal>run</goal>
+ </goals>
<configuration>
<target>
- <copy todir="${project.build.directory}/native/test/testData" overwrite="true">
- <fileset dir="${basedir}/src/main/native/testData" />
- </copy>
- <copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" />
- <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
- <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+ <copy todir="${project.build.directory}/native/test/testData"
+ overwrite="true">
+ <fileset dir="${basedir}/src/main/native/testData" />
+ </copy>
+ <copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" />
+ <exec executable="cmake" dir="${project.build.directory}/native"
+ failonerror="true">
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}" />
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
- <arg line="VERBOSE=1"/>
+ <arg line="VERBOSE=1" />
</exec>
- <!-- The second make is a workaround for HADOOP-9215. It can
- be removed when version 2.6 of cmake is no longer supported . -->
+ <!-- The second make is a workaround for HADOOP-9215. It can be
+ removed when version 2.6 of cmake is no longer supported . -->
<exec executable="make" dir="${project.build.directory}/native" failonerror="true"></exec>
- </target>
+ </target>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e788e8b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 0247a54..3094162 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -52,7 +52,7 @@ endfunction(dual_output_directory TGT DIR)
#
# This macro alters the behavior of find_package and find_library.
-# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
+# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
# You should save that variable before calling this function and restore it
# after you have accomplished your goal.
#
@@ -79,8 +79,8 @@ macro(set_find_shared_library_version LVERS)
endmacro(set_find_shared_library_version LVERS)
if (NOT GENERATED_JAVAH)
- #Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
+ #Must identify where the generated headers have been placed
+ MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
endif (NOT GENERATED_JAVAH)
find_package(JNI REQUIRED)
@@ -91,8 +91,8 @@ SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
# primitive configs
set(PRFLAGS "-DSIMPLE_MEMCPY")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -Wall")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
- -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
+set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
+ -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
set(D main/native/)
@@ -125,12 +125,12 @@ CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version("1")
-find_library(SNAPPY_LIBRARY
+find_library(SNAPPY_LIBRARY
NAMES snappy
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-find_path(SNAPPY_INCLUDE_DIR
+find_path(SNAPPY_INCLUDE_DIR
NAMES snappy.h
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
${CUSTOM_SNAPPY_INCLUDE})
@@ -147,15 +147,15 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
include_directories(
- ${GENERATED_JAVAH}
- ${D}
+ ${GENERATED_JAVAH}
+ ${D}
${D}/src
- ${D}/src/util
- ${D}/src/lib
- ${D}/test
+ ${D}/src/util
+ ${D}/src/lib
+ ${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
- #${CMAKE_CURRENT_SOURCE_DIR}/src
- #${CMAKE_BINARY_DIR}
+ #${CMAKE_CURRENT_SOURCE_DIR}/src
+ #${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
@@ -166,6 +166,13 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ # macosx does not have -lrt
+ set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+
add_dual_library(nativetask
${D}/lz4/lz4.c
${D}/cityhash/city.cc
@@ -208,60 +215,51 @@ add_dual_library(nativetask
${D}/src/util/Timer.cc
${D}/src/util/WritableUtils.cc
)
-target_link_libraries(nativetask
- #${LIB_DL}
- dl
- rt
- pthread
- z
- ${SNAPPY_LIBRARY}
- ${JAVA_JVM_LIBRARY}
-)
-add_executable(nttest
+target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
+
+add_executable(nttest
${D}/gtest/gtest-all.cc
- ${D}/test/lib/TestByteArray.cc
- ${D}/test/lib/TestByteBuffer.cc
- ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
- ${D}/test/lib/TestComparatorForStdSort.cc
- ${D}/test/lib/TestFixSizeContainer.cc
- ${D}/test/lib/TestMemoryPool.cc
- ${D}/test/lib/TestIterator.cc
- ${D}/test/lib/TestKVBuffer.cc
- ${D}/test/lib/TestMemBlockIterator.cc
- ${D}/test/lib/TestMemoryBlock.cc
- ${D}/test/lib/TestPartitionBucket.cc
- ${D}/test/lib/TestReadBuffer.cc
- ${D}/test/lib/TestReadWriteBuffer.cc
- ${D}/test/lib/TestTrackingCollector.cc
- ${D}/test/util/TestChecksum.cc
- ${D}/test/util/TestHash.cc
- ${D}/test/util/TestStringUtil.cc
- ${D}/test/util/TestSyncUtils.cc
- ${D}/test/util/TestWritableUtils.cc
- ${D}/test/TestCommand.cc
- ${D}/test/TestConfig.cc
- ${D}/test/TestCounter.cc
- ${D}/test/TestCompressions.cc
- ${D}/test/TestFileSystem.cc
- ${D}/test/TestIFile.cc
- ${D}/test/TestPrimitives.cc
- ${D}/test/TestSort.cc
- ${D}/test/TestMain.cc
+ ${D}/test/lib/TestByteArray.cc
+ ${D}/test/lib/TestByteBuffer.cc
+ ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
+ ${D}/test/lib/TestComparatorForStdSort.cc
+ ${D}/test/lib/TestFixSizeContainer.cc
+ ${D}/test/lib/TestMemoryPool.cc
+ ${D}/test/lib/TestIterator.cc
+ ${D}/test/lib/TestKVBuffer.cc
+ ${D}/test/lib/TestMemBlockIterator.cc
+ ${D}/test/lib/TestMemoryBlock.cc
+ ${D}/test/lib/TestPartitionBucket.cc
+ ${D}/test/lib/TestReadBuffer.cc
+ ${D}/test/lib/TestReadWriteBuffer.cc
+ ${D}/test/lib/TestTrackingCollector.cc
+ ${D}/test/util/TestChecksum.cc
+ ${D}/test/util/TestHash.cc
+ ${D}/test/util/TestStringUtil.cc
+ ${D}/test/util/TestSyncUtils.cc
+ ${D}/test/util/TestWritableUtils.cc
+ ${D}/test/TestCommand.cc
+ ${D}/test/TestConfig.cc
+ ${D}/test/TestCounter.cc
+ ${D}/test/TestCompressions.cc
+ ${D}/test/TestFileSystem.cc
+ ${D}/test/TestIFile.cc
+ ${D}/test/TestPrimitives.cc
+ ${D}/test/TestSort.cc
+ ${D}/test/TestMain.cc
${D}/test/test_commons.cc)
-set(CMAKE_EXE_LINKER_FLAGS "-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
-target_link_libraries(nttest
- nativetask_static
- dl
- rt
- pthread
- z
- ${SNAPPY_LIBRARY}
- # ${JAVA_JVM_LIBRARY}
+
+
+IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ # macos clang with libc++ does not have tr1/tuple, just tuple
+ SET_TARGET_PROPERTIES(nttest PROPERTIES COMPILE_FLAGS "-DGTEST_USE_OWN_TR1_TUPLE=1")
+ENDIF()
+
+target_link_libraries(nttest
+ nativetask_static
+ ${NT_DEPEND_LIBRARY}
)
-#if (NEED_LINK_DL)
-# set(LIB_DL dl)
-#endif (NEED_LINK_DL)
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
#
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e788e8b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
index 07b2cf1..89770ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
@@ -25,6 +25,7 @@
#include <stdlib.h>
#include <assert.h>
#include <string.h>
+#include <unistd.h>
#include <stdio.h>
#include <memory.h>
#include <fcntl.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e788e8b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 1e0bae4..02afbc0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -166,7 +166,7 @@ TEST(Perf, IFile) {
TEST(IFile, TestGlibCBug) {
std::string path("./testData/testGlibCBugSpill.out");
- uint32_t expect[5] = {-1538241715, -1288088794, -192294464, 563552421, 1661521654};
+ int32_t expect[5] = {-1538241715, -1288088794, -192294464, 563552421, 1661521654};
LOG("TestGlibCBug %s", path.c_str());
IFileSegment * segments = new IFileSegment [1];
@@ -182,7 +182,8 @@ TEST(IFile, TestGlibCBug) {
reader->nextPartition();
uint32_t index = 0;
while(NULL != (key = reader->nextKey(length))) {
- int realKey = bswap(*(uint32_t *)(key));
+ int32_t realKey = (int32_t)bswap(*(uint32_t *)(key));
+ ASSERT_LT(index, 5);
ASSERT_EQ(expect[index], realKey);
index++;
}
[42/50] [abbrv] git commit: Merge branch 'trunk' into MR-2841
Posted by ar...@apache.org.
Merge branch 'trunk' into MR-2841
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b160707a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b160707a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b160707a
Branch: refs/heads/HDFS-6581
Commit: b160707ace7900f360fa914cd291675c1d9703ec
Parents: 52a8b4d d1fa582
Author: Sean Zhong <cl...@gmail.com>
Authored: Sun Sep 7 13:01:27 2014 +0800
Committer: Sean Zhong <cl...@gmail.com>
Committed: Sun Sep 7 13:01:27 2014 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 13 +
.../src/main/bin/hadoop-functions.sh | 4 +-
.../crypto/key/kms/KMSClientProvider.java | 6 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++++++++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 88 +-----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 16 ++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +-
.../hadoop-hdfs/src/main/bin/hdfs | 15 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +
.../hadoop/hdfs/DistributedFileSystem.java | 24 ++
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/blockmanagement/BlockManager.java | 23 +-
.../server/blockmanagement/DatanodeManager.java | 6 +-
.../server/blockmanagement/HostFileManager.java | 4 +
.../server/datanode/ReplicaBeingWritten.java | 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../hdfs/server/namenode/FSNamesystem.java | 46 +--
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 175 +++++------
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43 +++
.../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24 +-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++++++++++++++++++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../TestValidateConfigurationSettings.java | 16 +-
.../namenode/ha/TestDelegationTokensWithHA.java | 63 ++--
.../hdfs/server/namenode/ha/TestHAMetrics.java | 10 +-
.../namenode/ha/TestHAStateTransitions.java | 52 ++--
.../namenode/ha/TestStandbyCheckpoints.java | 54 ++--
.../java/org/apache/hadoop/tools/TestTools.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 6 +
.../yarn/webapp/util/TestWebAppUtils.java | 4 +-
.../webapp/CrossOriginFilterInitializer.java | 12 +-
.../TestCrossOriginFilterInitializer.java | 7 +-
pom.xml | 1 +
42 files changed, 998 insertions(+), 373 deletions(-)
----------------------------------------------------------------------
[40/50] [abbrv] git commit: MAPREDUCE-6074. native-task: Fix release
audit warnings
Posted by ar...@apache.org.
MAPREDUCE-6074. native-task: Fix release audit warnings
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c91f9b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c91f9b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c91f9b1
Branch: refs/heads/HDFS-6581
Commit: 7c91f9b1484d487e792dca051fbd418697049422
Parents: 4b3f1e2
Author: Todd Lipcon <to...@apache.org>
Authored: Fri Sep 5 13:41:18 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 5 19:58:41 2014 -0700
----------------------------------------------------------------------
LICENSE.txt | 33 +++
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/LICENSE | 266 -------------------
.../hadoop-mapreduce-client-nativetask/pom.xml | 12 +-
.../mapred/nativetask/INativeComparable.java | 27 +-
.../src/main/native/COPYING | 87 ------
...oop_mapred_nativetask_NativeBatchProcessor.h | 54 ----
...che_hadoop_mapred_nativetask_NativeRuntime.h | 66 -----
.../src/main/native/test.sh | 11 +
.../nativetask/buffer/TestBufferPushPull.java | 13 +-
.../buffer/TestByteBufferReadWrite.java | 16 +-
.../nativetask/combinertest/CombinerTest.java | 4 +-
.../combinertest/OldAPICombinerTest.java | 5 +-
.../nativetask/combinertest/WordCount.java | 3 +-
.../nativetask/compresstest/CompressMapper.java | 4 +-
.../nativetask/compresstest/CompressTest.java | 2 +-
.../nativetask/handlers/TestCombineHandler.java | 13 +-
.../TestNativeCollectorOnlyHandler.java | 17 +-
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 3 +-
.../mapred/nativetask/kvtest/TestInputFile.java | 7 +-
.../nativetask/nonsorttest/NonSortTest.java | 2 +-
.../serde/TestNativeSerialization.java | 10 +-
.../nativetask/testutil/BytesFactory.java | 2 +-
.../mapred/nativetask/utils/TestBytesUtil.java | 20 +-
hadoop-mapreduce-project/pom.xml | 2 +-
25 files changed, 143 insertions(+), 537 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 946a6df..99989f1 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -288,3 +288,36 @@ lz4_encoder.h,lz4hc.h,lz4hc.c,lz4hc_encoder.h},
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
+
+
+For hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest
+---------------------------------------------------------------------
+Copyright 2008, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index ea17907..539e7be 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -22,3 +22,4 @@ MAPREDUCE-6056. native-task: move system test working dir to target dir and clea
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd)
+MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
deleted file mode 100644
index 0780741..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
+++ /dev/null
@@ -1,266 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-Licenses for third party projects used by this project:
-
-GoogleTest src/main/native/gtest
----------------------------------------------------------------------
-Copyright 2008, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-LZ4 src/main/native/lz4
----------------------------------------------------------------------
- LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011, Yann Collet.
- BSD License
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index f62743e..c18949e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -32,7 +32,6 @@
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -88,6 +87,17 @@
</includes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/main/native/testData/*</exclude>
+ <!-- gtest has a compatible license -->
+ <exclude>src/main/native/gtest/**/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
index e69248f..1ec05db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
@@ -22,34 +22,33 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
+ * Any key type that is comparable at native side must implement this interface.
*
- * Any key type that is comparable at native side must implement this interface
- *
- * a native comparator function should have the ComparatorPtr type
- *
+ * A native comparator function should have the ComparatorPtr type:
+ * <code>
* typedef int (*ComparatorPtr)(const char * src, uint32_t srcLength,
* const char * dest, uint32_t destLength);
- *
- * keys are in serialized format at native side. The function has passed in
+ * </code>
+ * Keys are in serialized format at native side. The function has passed in
* the keys' locations and lengths such that we can compare them in the same
- * logic as their Java comparator
+ * logic as their Java comparator.
*
+ * For example, a HiveKey serialized as an int field (containing the length of
+ * raw bytes) + raw bytes.
+ * When comparing two HiveKeys, we first read the length field and then
+ * compare the raw bytes by invoking the BytesComparator provided by our library.
+ * We pass the location and length of raw bytes into BytesComparator.
*
- * For example, a HiveKey {@see HiveKey#write} is serialized as
- * int field (containing the length of raw bytes) + raw bytes
- * When comparing two HiveKeys, we firstly read the length field and then
- * comparing the raw bytes invoking the BytesComparator provided by our library.
- * We pass the location and length of raw bytes into BytesComparator
- *
+ * <code>
* int HivePlatform::HiveKeyComparator(const char * src, uint32_t srcLength,
* const char * dest, uint32_t destLength) {
* uint32_t sl = bswap(*(uint32_t*)src);
* uint32_t dl = bswap(*(uint32_t*)dest);
* return NativeObjectFactory::BytesComparator(src + 4, sl, dest + 4, dl);
* }
+ * </code>
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface INativeComparable {
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
deleted file mode 100644
index 1d60d1d..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
+++ /dev/null
@@ -1,87 +0,0 @@
-CityHash
----------------------------------------------------------------------
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-GoogleTest
----------------------------------------------------------------------
-Copyright 2008, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-LZ4
----------------------------------------------------------------------
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011-2014, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h
deleted file mode 100644
index 1dd02ab..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* DO NOT EDIT THIS FILE - it is machine generated */
-#include <jni.h>
-/* Header for class org_apache_hadoop_mapred_nativetask_NativeBatchProcessor */
-
-#ifndef _Included_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
-#define _Included_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: setupHandler
- * Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler(
- JNIEnv *, jobject, jlong, jobjectArray configs);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: nativeProcessInput
- * Signature: (JI)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_nativeProcessInput(
- JNIEnv *, jobject, jlong, jint);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: nativeFinish
- * Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_nativeFinish(
- JNIEnv *, jobject, jlong);
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: nativeCommand
- * Signature: (J[B)[B
- */JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_nativeCommand(
- JNIEnv *, jobject, jlong, jint, jbyteArray);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: nativeLoadData
- * Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_nativeLoadData(
- JNIEnv *, jobject, jlong);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeBatchProcessor
- * Method: InitIDs
- * Signature: ()V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_InitIDs(
- JNIEnv *, jclass);
-
-#ifdef __cplusplus
-}
-#endif
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
deleted file mode 100644
index 915c532..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/* DO NOT EDIT THIS FILE - it is machine generated */
-#include <jni.h>
-/* Header for class org_apache_hadoop_mapred_nativetask_NativeRuntime */
-
-#ifndef _Included_org_apache_hadoop_mapred_nativetask_NativeRuntime
-#define _Included_org_apache_hadoop_mapred_nativetask_NativeRuntime
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: supportCompressionCodec
- * Signature: ([B)Z
- */
-JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
- (JNIEnv *, jclass, jbyteArray);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNIRelease
- * Signature: ()V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(JNIEnv *,
- jclass);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNIConfigure
- * Signature: ([[B)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure(
- JNIEnv *, jclass, jobjectArray);
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNICreateNativeObject
- * Signature: ([B)J
- */JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNativeObject(
- JNIEnv *, jclass, jbyteArray);
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNICreateDefaultNativeObject
- * Signature: ([B)J
- */JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject(
- JNIEnv *, jclass, jbyteArray);
-
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNIReleaseNativeObject
- * Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject(
- JNIEnv *, jclass, jlong);
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNIRegisterModule
- * Signature: ([B[B)I
- */JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule(
- JNIEnv *, jclass, jbyteArray, jbyteArray);
-/*
- * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
- * Method: JNIUpdateStatus
- * Signature: ()[B
- */JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus(
- JNIEnv *, jclass);
-
-#ifdef __cplusplus
-}
-#endif
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
index 01506e6..fe1ca78 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
@@ -1,4 +1,15 @@
#!/bin/sh
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
# only do normal tests by default
FILTER="--gtest_filter=-Perf.*"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
index 5dcac35..48c32c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
@@ -22,9 +22,6 @@ import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.mapred.RawKeyValueIterator;
@@ -42,21 +39,24 @@ import org.apache.hadoop.mapred.nativetask.testutil.TestInput;
import org.apache.hadoop.mapred.nativetask.testutil.TestInput.KV;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.util.Progress;
+
+import org.junit.Assert;
import org.junit.Before;
+import org.junit.Test;
-@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" })
-public class TestBufferPushPull extends TestCase {
+@SuppressWarnings({ "rawtypes", "unchecked"})
+public class TestBufferPushPull {
public static int BUFFER_LENGTH = 100; // 100 bytes
public static int INPUT_KV_COUNT = 1000;
private KV<BytesWritable, BytesWritable>[] dataInput;
- @Override
@Before
public void setUp() {
this.dataInput = TestInput.getMapInputs(INPUT_KV_COUNT);
}
+ @Test
public void testPush() throws Exception {
final byte[] buff = new byte[BUFFER_LENGTH];
@@ -99,6 +99,7 @@ public class TestBufferPushPull extends TestCase {
pushee.close();
}
+ @Test
public void testPull() throws Exception {
final byte[] buff = new byte[BUFFER_LENGTH];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
index a405634..106fdcb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
@@ -23,11 +23,13 @@ import com.google.common.base.Charsets;
import com.google.common.primitives.Shorts;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
-import junit.framework.Assert;
-import junit.framework.TestCase;
+import org.junit.Assert;
+import org.junit.Test;
+
import org.mockito.Mockito;
-public class TestByteBufferReadWrite extends TestCase {
+public class TestByteBufferReadWrite {
+ @Test
public void testReadWrite() throws IOException {
byte[] buff = new byte[10000];
@@ -89,6 +91,7 @@ public class TestByteBufferReadWrite extends TestCase {
* Test that Unicode characters outside the basic multilingual plane,
* such as this cat face, are properly encoded.
*/
+ @Test
public void testCatFace() throws IOException {
byte[] buff = new byte[10];
MockDataTarget target = new MockDataTarget(buff);
@@ -100,13 +103,14 @@ public class TestByteBufferReadWrite extends TestCase {
InputBuffer input = new InputBuffer(buff);
input.rewind(0, buff.length);
ByteBufferDataReader reader = new ByteBufferDataReader(input);
- assertEquals(catFace, reader.readUTF());
+ Assert.assertEquals(catFace, reader.readUTF());
// Check that the standard Java one can read it too
String fromJava = new java.io.DataInputStream(new ByteArrayInputStream(buff)).readUTF();
- assertEquals(catFace, fromJava);
+ Assert.assertEquals(catFace, fromJava);
}
+ @Test
public void testShortOfSpace() throws IOException {
byte[] buff = new byte[10];
MockDataTarget target = new MockDataTarget(buff);
@@ -120,6 +124,8 @@ public class TestByteBufferReadWrite extends TestCase {
Assert.assertEquals(true, writer.shortOfSpace(100));
}
+
+ @Test
public void testFlush() throws IOException {
byte[] buff = new byte[10];
MockDataTarget target = Mockito.spy(new MockDataTarget(buff));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index 3aa8a07..cf6e286 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -106,10 +106,10 @@ public class CombinerTest {
conf.set("fileoutputpath", outputpath);
final FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputpath))) {
- fs.delete(new Path(outputpath));
+ fs.delete(new Path(outputpath), true);
}
fs.close();
- final Job job = new Job(conf, jobname);
+ final Job job = Job.getInstance(conf, jobname);
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
index c3fc02f..8954b52 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
@@ -60,7 +61,7 @@ public class OldAPICombinerTest {
RunningJob nativeRunning = JobClient.runJob(nativeJob);
Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(
- Task.Counter.REDUCE_INPUT_RECORDS);
+ TaskCounter.REDUCE_INPUT_RECORDS);
final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
@@ -70,7 +71,7 @@ public class OldAPICombinerTest {
RunningJob normalRunning = JobClient.runJob(normalJob);
Counter normalReduceGroups = normalRunning.getCounters().findCounter(
- Task.Counter.REDUCE_INPUT_RECORDS);
+ TaskCounter.REDUCE_INPUT_RECORDS);
final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
index 7b33672..c022848 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
@@ -76,7 +76,8 @@ public class WordCount {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
- final Job job = new Job(conf, conf.get(MRJobConfig.JOB_NAME, "word count"));
+ final Job job = Job.getInstance(conf,
+ conf.get(MRJobConfig.JOB_NAME, "word count"));
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
index 8198b47..a0de949 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
@@ -44,7 +44,7 @@ public class CompressMapper {
public static Job getCompressJob(String jobname, Configuration conf,
String inputpath, String outputpath)
throws Exception {
- Job job = new Job(conf, jobname + "-CompressMapperJob");
+ Job job = Job.getInstance(conf, jobname + "-CompressMapperJob");
job.setJarByClass(CompressMapper.class);
job.setMapperClass(TextCompressMapper.class);
job.setOutputKeyClass(Text.class);
@@ -52,7 +52,7 @@ public class CompressMapper {
// if output file exists ,delete it
final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
if (hdfs.exists(new Path(outputpath))) {
- hdfs.delete(new Path(outputpath));
+ hdfs.delete(new Path(outputpath), true);
}
hdfs.close();
job.setInputFormatClass(SequenceFileInputFormat.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index b8f9dfc..ca39fa5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -124,7 +124,7 @@ public class CompressTest {
final ScenarioConfiguration conf = new ScenarioConfiguration();
final FileSystem fs = FileSystem.get(conf);
final Path path = new Path(TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR);
- fs.delete(path);
+ fs.delete(path, true);
if (!fs.exists(path)) {
new TestInputFile(hadoopConf.getInt(
TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
index 81d632d..68499a7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
@@ -19,19 +19,19 @@ package org.apache.hadoop.mapred.nativetask.handlers;
import java.io.IOException;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
import org.apache.hadoop.mapred.Task.CombinerRunner;
import org.apache.hadoop.mapred.nativetask.Command;
import org.apache.hadoop.mapred.nativetask.INativeHandler;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;
@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" })
-public class TestCombineHandler extends TestCase {
+public class TestCombineHandler {
private CombinerHandler handler;
private INativeHandler nativeHandler;
@@ -39,7 +39,7 @@ public class TestCombineHandler extends TestCase {
private BufferPuller puller;
private CombinerRunner combinerRunner;
- @Override
+ @Before
public void setUp() throws IOException {
this.nativeHandler = Mockito.mock(INativeHandler.class);
this.pusher = Mockito.mock(BufferPusher.class);
@@ -50,12 +50,14 @@ public class TestCombineHandler extends TestCase {
new InputBuffer(BufferType.HEAP_BUFFER, 100));
}
+ @Test
public void testCommandDispatcherSetting() throws IOException {
this.handler = new CombinerHandler(nativeHandler, combinerRunner, puller, pusher);
Mockito.verify(nativeHandler, Mockito.times(1)).setCommandDispatcher(Matchers.eq(handler));
Mockito.verify(nativeHandler, Mockito.times(1)).setDataReceiver(Matchers.eq(puller));
}
+ @Test
public void testCombine() throws IOException, InterruptedException, ClassNotFoundException {
this.handler = new CombinerHandler(nativeHandler, combinerRunner, puller, pusher);
Assert.assertEquals(null, handler.onCall(CombinerHandler.COMBINE, null));
@@ -70,6 +72,7 @@ public class TestCombineHandler extends TestCase {
Mockito.verify(nativeHandler, Mockito.times(1)).close();
}
+ @Test
public void testOnCall() throws IOException {
this.handler = new CombinerHandler(nativeHandler, combinerRunner, puller, pusher);
Assert.assertEquals(null, handler.onCall(new Command(-1), null));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
index 6c15360..1c8bf7a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
@@ -19,9 +19,6 @@ package org.apache.hadoop.mapred.nativetask.handlers;
import java.io.IOException;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -36,11 +33,15 @@ import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapred.nativetask.util.OutputUtil;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;
@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" })
-public class TestNativeCollectorOnlyHandler extends TestCase {
+public class TestNativeCollectorOnlyHandler {
private NativeCollectorOnlyHandler handler;
private INativeHandler nativeHandler;
@@ -49,7 +50,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
private TaskContext taskContext;
private static final String LOCAL_DIR = TestConstants.NATIVETASK_TEST_DIR + "/local";
- @Override
+ @Before
public void setUp() throws IOException {
this.nativeHandler = Mockito.mock(INativeHandler.class);
this.pusher = Mockito.mock(BufferPusher.class);
@@ -69,12 +70,12 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
new InputBuffer(BufferType.HEAP_BUFFER, 100));
}
- @Override
+ @After
public void tearDown() throws IOException {
FileSystem.getLocal(new Configuration()).delete(new Path(LOCAL_DIR));
}
-
+ @Test
public void testCollect() throws IOException {
this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
handler.collect(new BytesWritable(), new BytesWritable(), 100);
@@ -89,6 +90,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
Mockito.verify(nativeHandler, Mockito.times(1)).close();
}
+ @Test
public void testGetCombiner() throws IOException {
this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
Mockito.when(combiner.getId()).thenReturn(100L);
@@ -97,6 +99,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
Assert.assertEquals(100L, result.readLong());
}
+ @Test
public void testOnCall() throws IOException {
this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
boolean thrown = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 0a4c955..2d4515f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -61,6 +61,7 @@ public class KVJob {
public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
@Override
+ @SuppressWarnings({"unchecked"})
public void reduce(KTYPE key, Iterable<VTYPE> values, Context context)
throws IOException, InterruptedException {
long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
@@ -79,7 +80,7 @@ public class KVJob {
public KVJob(String jobname, Configuration conf,
Class<?> keyclass, Class<?> valueclass,
String inputpath, String outputpath) throws Exception {
- job = new Job(conf, jobname);
+ job = Job.getInstance(conf, jobname);
job.setJarByClass(KVJob.class);
job.setMapperClass(KVJob.ValueMapper.class);
job.setOutputKeyClass(keyclass);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index 49331ce..04db67a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -141,8 +141,11 @@ public class TestInputFile {
try {
final Path outputfilepath = new Path(filepath);
final ScenarioConfiguration conf= new ScenarioConfiguration();
- final FileSystem hdfs = outputfilepath.getFileSystem(conf);
- writer = new SequenceFile.Writer(hdfs, conf, outputfilepath, tmpkeycls, tmpvaluecls);
+ writer = SequenceFile.createWriter(
+ conf,
+ SequenceFile.Writer.file(outputfilepath),
+ SequenceFile.Writer.keyClass(tmpkeycls),
+ SequenceFile.Writer.valueClass(tmpvaluecls));
} catch (final Exception e) {
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index f81d94f..81e1eae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -101,7 +101,7 @@ public class NonSortTest {
fs.delete(new Path(outputpath), true);
}
fs.close();
- final Job job = new Job(conf, jobName);
+ final Job job = Job.getInstance(conf, jobName);
job.setJarByClass(NonSortTestMR.class);
job.setMapperClass(NonSortTestMR.Map.class);
job.setReducerClass(NonSortTestMR.KeyHashSumReduce.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
index cf708a1..5666cc3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
@@ -21,14 +21,14 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+import org.junit.Assert;
+import org.junit.Test;
-@SuppressWarnings({ "rawtypes", "deprecation" })
-public class TestNativeSerialization extends TestCase {
+@SuppressWarnings({ "rawtypes" })
+public class TestNativeSerialization {
+ @Test
public void testRegisterAndGet() throws IOException {
final NativeSerialization serialization = NativeSerialization.getInstance();
serialization.reset();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
index bee2f0c..4df48fc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
+@SuppressWarnings("deprecation")
public class BytesFactory {
public static Random r = new Random();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
index c09f08c..5bafa43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
@@ -19,13 +19,15 @@ package org.apache.hadoop.mapred.nativetask.utils;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
-import junit.framework.Assert;
-import junit.framework.TestCase;
+
+import org.junit.Assert;
+import org.junit.Test;
import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-@SuppressWarnings({ "deprecation" })
-public class TestBytesUtil extends TestCase {
+public class TestBytesUtil {
+
+ @Test
public void testBytesIntConversion() {
final int a = 1000;
final byte[] intBytes = Ints.toByteArray(a);
@@ -33,6 +35,7 @@ public class TestBytesUtil extends TestCase {
Assert.assertEquals(a, BytesUtil.toInt(intBytes, 0));
}
+ @Test
public void testBytesLongConversion() {
final long l = 1000000L;
final byte[] longBytes = Longs.toByteArray(l);
@@ -40,20 +43,23 @@ public class TestBytesUtil extends TestCase {
Assert.assertEquals(l, BytesUtil.toLong(longBytes, 0));
}
+ @Test
public void testBytesFloatConversion() {
final float f = 3.14f;
final byte[] floatBytes = BytesUtil.toBytes(f);
- Assert.assertEquals(f, BytesUtil.toFloat(floatBytes));
+ Assert.assertEquals(f, BytesUtil.toFloat(floatBytes), 0.0f);
}
+ @Test
public void testBytesDoubleConversion() {
final double d = 3.14;
final byte[] doubleBytes = BytesUtil.toBytes(d);
- Assert.assertEquals(d, BytesUtil.toDouble(doubleBytes));
+ Assert.assertEquals(d, BytesUtil.toDouble(doubleBytes), 0.0);
}
-
+
+ @Test
public void testToStringBinary() {
Assert.assertEquals("\\x01\\x02ABC",
BytesUtil.toStringBinary(new byte[] { 1, 2, 65, 66, 67 }));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 9110942..5cee534 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -194,7 +194,7 @@
<configuration>
<excludes>
<exclude>.eclipse.templates/</exclude>
- <exclude>CHANGES.txt</exclude>
+ <exclude>CHANGES.*txt</exclude>
<exclude>lib/jdiff/**</exclude>
</excludes>
</configuration>
[33/50] [abbrv] git commit: MAPREDUCE-6055. native-task: findbugs,
interface annotations, and other misc cleanup
Posted by ar...@apache.org.
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1081d9ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1081d9ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1081d9ce
Branch: refs/heads/HDFS-6581
Commit: 1081d9cee23aa661d7c9165bc9855865a38b528e
Parents: cce7d1e
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Sep 3 12:02:47 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Sep 3 12:22:38 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/Command.java | 8 +++++
.../mapred/nativetask/CommandDispatcher.java | 11 ++-----
.../hadoop/mapred/nativetask/Constants.java | 18 ++++++-----
.../hadoop/mapred/nativetask/DataChannel.java | 3 ++
.../hadoop/mapred/nativetask/DataReceiver.java | 3 ++
.../mapred/nativetask/HadoopPlatform.java | 6 ++--
.../mapred/nativetask/ICombineHandler.java | 3 ++
.../mapred/nativetask/INativeComparable.java | 5 +++
.../mapred/nativetask/INativeHandler.java | 2 ++
.../mapred/nativetask/NativeBatchProcessor.java | 3 +-
.../mapred/nativetask/NativeDataSource.java | 10 ++----
.../mapred/nativetask/NativeDataTarget.java | 14 +++------
.../NativeMapOutputCollectorDelegator.java | 11 ++++---
.../hadoop/mapred/nativetask/NativeRuntime.java | 10 +++---
.../hadoop/mapred/nativetask/Platform.java | 15 ++++-----
.../hadoop/mapred/nativetask/Platforms.java | 7 +++--
.../mapred/nativetask/StatusReportChecker.java | 33 ++++++++++----------
.../hadoop/mapred/nativetask/TaskContext.java | 29 +++++++++--------
.../mapred/nativetask/buffer/BufferType.java | 5 +--
.../nativetask/buffer/ByteBufferDataReader.java | 10 ++----
.../nativetask/buffer/ByteBufferDataWriter.java | 10 +++---
.../nativetask/buffer/DataInputStream.java | 2 ++
.../nativetask/buffer/DataOutputStream.java | 11 +++----
.../mapred/nativetask/buffer/InputBuffer.java | 2 ++
.../mapred/nativetask/buffer/OutputBuffer.java | 2 ++
.../nativetask/handlers/BufferPullee.java | 4 ++-
.../nativetask/handlers/BufferPuller.java | 26 ++++++++++++---
.../nativetask/handlers/BufferPushee.java | 2 ++
.../nativetask/handlers/BufferPusher.java | 3 +-
.../nativetask/handlers/CombinerHandler.java | 19 +++++------
.../mapred/nativetask/handlers/IDataLoader.java | 3 ++
.../handlers/NativeCollectorOnlyHandler.java | 22 ++++++++-----
.../serde/BoolWritableSerializer.java | 2 ++
.../serde/ByteWritableSerializer.java | 2 ++
.../serde/BytesWritableSerializer.java | 2 ++
.../nativetask/serde/DefaultSerializer.java | 4 +++
.../serde/DoubleWritableSerializer.java | 2 ++
.../serde/FloatWritableSerializer.java | 3 ++
.../mapred/nativetask/serde/IKVSerializer.java | 12 ++++---
.../nativetask/serde/INativeSerializer.java | 4 +++
.../nativetask/serde/IntWritableSerializer.java | 2 ++
.../mapred/nativetask/serde/KVSerializer.java | 17 ++++++----
.../serde/LongWritableSerializer.java | 2 ++
.../nativetask/serde/NativeSerialization.java | 2 ++
.../serde/NullWritableSerializer.java | 2 ++
.../serde/SerializationFramework.java | 3 ++
.../mapred/nativetask/serde/TextSerializer.java | 2 ++
.../serde/VIntWritableSerializer.java | 3 +-
.../serde/VLongWritableSerializer.java | 3 +-
.../mapred/nativetask/util/BytesUtil.java | 2 ++
.../mapred/nativetask/util/ConfigUtil.java | 5 ++-
.../nativetask/util/LocalJobOutputFiles.java | 2 ++
.../nativetask/util/NativeTaskOutput.java | 2 ++
.../nativetask/util/NativeTaskOutputFiles.java | 3 +-
.../mapred/nativetask/util/OutputUtil.java | 5 +--
.../mapred/nativetask/util/ReadWriteBuffer.java | 5 +--
.../mapred/nativetask/util/SizedWritable.java | 4 ++-
.../mapred/nativetask/TestTaskContext.java | 4 +--
59 files changed, 264 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 7c9558e..269a2f6 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -19,3 +19,4 @@ MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
MAPREDUCE-6054. native-task: Speed up tests (todd)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
+MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java
index 80a5658..6256fd1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.mapred.nativetask;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
public class Command {
private int id;
@@ -46,4 +49,9 @@ public class Command {
}
return false;
}
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java
index 52af730..7984794 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java
@@ -19,20 +19,15 @@
package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
/**
* a CommandDispatcher receives {@link Command} from upstream
* and performs corresponding operations
*/
+@InterfaceAudience.Private
public interface CommandDispatcher {
-
- /**
- *
- * @param command
- * @param parameter
- * @return
- * @throws IOException
- */
public ReadWriteBuffer onCall(Command command, ReadWriteBuffer parameter) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
index e71326f..ff1a95c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
@@ -15,9 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.mapred.nativetask;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
public class Constants {
public static final String MAP_SORT_CLASS = "map.sort.class";
@@ -40,17 +42,17 @@ public class Constants {
public static final String NATIVE_OUTPUT_FILE_NAME = "native.output.file.name";
public static final String NATIVE_PROCESSOR_BUFFER_KB = "native.processor.buffer.kb";
- public static int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64;
- public static int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024;
+ public static final int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64;
+ public static final int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024;
public static final String NATIVE_STATUS_UPDATE_INTERVAL = "native.update.interval";
- public static int NATIVE_STATUS_UPDATE_INTERVAL_DEFVAL = 3000;
+ public static final int NATIVE_STATUS_UPDATE_INTERVAL_DEFVAL = 3000;
public static final String SERIALIZATION_FRAMEWORK = "SerializationFramework";
- public static int SIZEOF_PARTITION_LENGTH = 4;
- public static int SIZEOF_KEY_LENGTH = 4;
- public static int SIZEOF_VALUE_LENGTH = 4;
- public static int SIZEOF_KV_LENGTH = SIZEOF_KEY_LENGTH + SIZEOF_VALUE_LENGTH;
+ public static final int SIZEOF_PARTITION_LENGTH = 4;
+ public static final int SIZEOF_KEY_LENGTH = 4;
+ public static final int SIZEOF_VALUE_LENGTH = 4;
+ public static final int SIZEOF_KV_LENGTH = SIZEOF_KEY_LENGTH + SIZEOF_VALUE_LENGTH;
public static final String NATIVE_CLASS_LIBRARY = "native.class.library";
public static final String NATIVE_CLASS_LIBRARY_CUSTOM = "native.class.library.custom";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataChannel.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataChannel.java
index e8132bd..03469ee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataChannel.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataChannel.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.mapred.nativetask;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
public enum DataChannel {
/**
* We will only read data from this channel
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
index c47cdac..85f78ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
/**
* a DataReceiver pulls in arriving data, an example
* is {@link org.apache.hadoop.mapred.nativetask.handlers.BufferPuller}
*/
+@InterfaceAudience.Private
public interface DataReceiver {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
index 7599bb8..da6f252 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -36,6 +37,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.*;
+@InterfaceAudience.Private
public class HadoopPlatform extends Platform {
private static final Log LOG = LogFactory.getLog(HadoopPlatform.class);
@@ -61,7 +63,7 @@ public class HadoopPlatform extends Platform {
}
@Override
- public boolean support(String keyClassName, INativeSerializer serializer, JobConf job) {
+ public boolean support(String keyClassName, INativeSerializer<?> serializer, JobConf job) {
if (keyClassNames.contains(keyClassName)
&& serializer instanceof INativeComparable) {
return true;
@@ -71,7 +73,7 @@ public class HadoopPlatform extends Platform {
}
@Override
- public boolean define(Class comparatorClass) {
+ public boolean define(Class<?> comparatorClass) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
index 8f50863..e4e706b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
@@ -19,9 +19,12 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
/**
* interacts with native side to support Java Combiner
*/
+@InterfaceAudience.Private
public interface ICombineHandler {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
index ae07f3b..e69248f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.mapred.nativetask;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
/**
*
* Any key type that is comparable at native side must implement this interface
@@ -45,6 +48,8 @@ package org.apache.hadoop.mapred.nativetask;
* return NativeObjectFactory::BytesComparator(src + 4, sl, dest + 4, dl);
* }
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public interface INativeComparable {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
index 3e6e3ac..9506c7f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
@@ -19,12 +19,14 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
/**
* A Handler accept input, and give output can be used to transfer command and data
*/
+@InterfaceAudience.Private
public interface INativeHandler extends NativeDataTarget, NativeDataSource {
public String name();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index 837da0e..3c3811f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
@@ -30,11 +31,11 @@ import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
-import org.apache.hadoop.util.DirectBufferPool;
/**
* used to create channel, transfer data and command between Java and native
*/
+@InterfaceAudience.Private
public class NativeBatchProcessor implements INativeHandler {
private static Log LOG = LogFactory.getLog(NativeBatchProcessor.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataSource.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataSource.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataSource.java
index d0bfa93..f802b3f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataSource.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataSource.java
@@ -20,33 +20,27 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
-import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
/**
* NativeDataSource loads data from upstream
*/
+@InterfaceAudience.Private
public interface NativeDataSource {
/**
* get input buffer
- *
- * @return
*/
public InputBuffer getInputBuffer();
/**
* set listener. When data from upstream arrives, the listener will be activated.
- *
- * @param handler
*/
void setDataReceiver(DataReceiver handler);
/**
* load data from upstream
- *
- * @return
- * @throws IOException
*/
public void loadData() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataTarget.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataTarget.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataTarget.java
index 7a780eb..d91070c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataTarget.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeDataTarget.java
@@ -20,31 +20,27 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer;
/**
* NativeDataTarge sends data to downstream
*/
+@InterfaceAudience.Private
public interface NativeDataTarget {
/**
- * send a signal to indicate that the data has been stored in output buffer
- *
- * @throws IOException
+ * Sends a signal to indicate that the data has been stored in output buffer
*/
public void sendData() throws IOException;
/**
- * Send a signal that there is no more data
- *
- * @throws IOException
+ * Sends a signal that there is no more data
*/
public void finishSendData() throws IOException;
/**
- * get the output buffer.
- *
- * @return
+ * Gets the output buffer.
*/
public OutputBuffer getOutputBuffer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index beb3851..224b95b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -17,13 +17,13 @@
*/
package org.apache.hadoop.mapred.nativetask;
-import java.io.File;
import java.io.IOException;
import com.google.common.base.Charsets;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapred.JobConf;
@@ -34,13 +34,12 @@ import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.QuickSort;
-import org.apache.hadoop.util.RunJar;
/**
* native map output collector wrapped in Java interface
*/
+@InterfaceAudience.Private
public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollector<K, V> {
private static Log LOG = LogFactory.getLog(NativeMapOutputCollectorDelegator.class);
@@ -67,6 +66,7 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
handler.flush();
}
+ @SuppressWarnings("unchecked")
@Override
public void init(Context context) throws IOException, ClassNotFoundException {
this.job = context.getJobConf();
@@ -79,7 +79,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
throw new InvalidJobConfException(message);
}
- Class comparatorClass = job.getClass(MRJobConfig.KEY_COMPARATOR, null, RawComparator.class);
+ Class<?> comparatorClass = job.getClass(MRJobConfig.KEY_COMPARATOR, null,
+ RawComparator.class);
if (comparatorClass != null && !Platforms.define(comparatorClass)) {
String message = "Native output collector doesn't support customized java comparator "
+ job.get(MRJobConfig.KEY_COMPARATOR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index 5ce3c89..395cc89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import com.google.common.base.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.FloatWritable;
@@ -34,11 +35,12 @@ import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
import org.apache.hadoop.util.VersionInfo;
/**
- * This class stands for the native runtime It has three functions: 1. Create native handlers for map, reduce,
- * outputcollector, and etc 2. Configure native task with provided MR configs 3. Provide file system api to native
- * space, so that it can use File system like HDFS.
- *
+ * This class stands for the native runtime It has three functions:
+ * 1. Create native handlers for map, reduce, outputcollector, etc
+ * 2. Configure native task with provided MR configs
+ * 3. Provide file system api to native space, so that it can use File system like HDFS.
*/
+@InterfaceAudience.Private
public class NativeRuntime {
private static Log LOG = LogFactory.getLog(NativeRuntime.class);
private static boolean nativeLibraryLoaded = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
index aef53ce..7e41989 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.InvalidJobConfException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
@@ -39,6 +37,8 @@ import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
* that supports all key types of Hadoop and users could implement their custom
* platform.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public abstract class Platform {
private final NativeSerialization serialization;
protected Set<String> keyClassNames = new HashSet<String>();
@@ -67,7 +67,7 @@ public abstract class Platform {
* @param key key serializer class
* @throws IOException
*/
- protected void registerKey(String keyClassName, Class key) throws IOException {
+ protected void registerKey(String keyClassName, Class<?> key) throws IOException {
serialization.register(keyClassName, key);
keyClassNames.add(keyClassName);
}
@@ -85,7 +85,8 @@ public abstract class Platform {
* @return true if the platform has implemented native comparators of the key and
* false otherwise
*/
- protected abstract boolean support(String keyClassName, INativeSerializer serializer, JobConf job);
+ protected abstract boolean support(String keyClassName,
+ INativeSerializer<?> serializer, JobConf job);
/**
@@ -98,5 +99,5 @@ public abstract class Platform {
* @param keyComparator comparator set with mapreduce.job.output.key.comparator.class
* @return
*/
- protected abstract boolean define(Class keyComparator);
+ protected abstract boolean define(Class<?> keyComparator);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
index d0a8496..9fad3a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
@@ -22,6 +22,7 @@ import java.util.ServiceLoader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
@@ -33,6 +34,7 @@ import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
* it is also the facade to check for key type support and other
* platform methods
*/
+@InterfaceAudience.Private
public class Platforms {
private static final Log LOG = LogFactory.getLog(Platforms.class);
@@ -48,7 +50,8 @@ public class Platforms {
}
}
- public static boolean support(String keyClassName, INativeSerializer serializer, JobConf job) {
+ public static boolean support(String keyClassName,
+ INativeSerializer<?> serializer, JobConf job) {
synchronized (platforms) {
for (Platform platform : platforms) {
if (platform.support(keyClassName, serializer, job)) {
@@ -61,7 +64,7 @@ public class Platforms {
return false;
}
- public static boolean define(Class keyComparator) {
+ public static boolean define(Class<?> keyComparator) {
synchronized (platforms) {
for (Platform platform : platforms) {
if (platform.define(keyComparator)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
index 1fa59dc..f152074 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
@@ -21,17 +21,18 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.Task.Counter;
import org.apache.hadoop.mapred.Task.TaskReporter;
+import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
/**
* Will periodically check status from native and report to MR framework.
*
*/
-public class StatusReportChecker implements Runnable {
+class StatusReportChecker implements Runnable {
private static Log LOG = LogFactory.getLog(StatusReportChecker.class);
- public static int INTERVAL = 1000; // milli-seconds
+ public static final int INTERVAL = 1000; // milliseconds
private Thread checker;
private final TaskReporter reporter;
@@ -68,19 +69,19 @@ public class StatusReportChecker implements Runnable {
}
protected void initUsedCounters() {
- reporter.getCounter(Counter.MAP_INPUT_RECORDS);
- reporter.getCounter(Counter.MAP_OUTPUT_RECORDS);
- reporter.getCounter(Counter.MAP_INPUT_BYTES);
- reporter.getCounter(Counter.MAP_OUTPUT_BYTES);
- reporter.getCounter(Counter.MAP_OUTPUT_MATERIALIZED_BYTES);
- reporter.getCounter(Counter.COMBINE_INPUT_RECORDS);
- reporter.getCounter(Counter.COMBINE_OUTPUT_RECORDS);
- reporter.getCounter(Counter.REDUCE_INPUT_RECORDS);
- reporter.getCounter(Counter.REDUCE_OUTPUT_RECORDS);
- reporter.getCounter(Counter.REDUCE_INPUT_GROUPS);
- reporter.getCounter(Counter.SPILLED_RECORDS);
- reporter.getCounter(Counter.MAP_OUTPUT_BYTES);
- reporter.getCounter(Counter.MAP_OUTPUT_RECORDS);
+ reporter.getCounter(TaskCounter.MAP_INPUT_RECORDS);
+ reporter.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
+ reporter.getCounter(FileInputFormatCounter.BYTES_READ);
+ reporter.getCounter(TaskCounter.MAP_OUTPUT_BYTES);
+ reporter.getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES);
+ reporter.getCounter(TaskCounter.COMBINE_INPUT_RECORDS);
+ reporter.getCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
+ reporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);
+ reporter.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS);
+ reporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
+ reporter.getCounter(TaskCounter.SPILLED_RECORDS);
+ reporter.getCounter(TaskCounter.MAP_OUTPUT_BYTES);
+ reporter.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
}
public synchronized void start() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/TaskContext.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/TaskContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/TaskContext.java
index a4430ed..555ced1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/TaskContext.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/TaskContext.java
@@ -17,20 +17,23 @@
*/
package org.apache.hadoop.mapred.nativetask;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Task.TaskReporter;
import org.apache.hadoop.mapred.TaskAttemptID;
+@InterfaceAudience.Private
public class TaskContext {
private final JobConf conf;
- private Class iKClass;
- private Class iVClass;
- private Class oKClass;
- private Class oVClass;
+ private Class<?> iKClass;
+ private Class<?> iVClass;
+ private Class<?> oKClass;
+ private Class<?> oVClass;
private final TaskReporter reporter;
private final TaskAttemptID taskAttemptID;
- public TaskContext(JobConf conf, Class iKClass, Class iVClass, Class oKClass, Class oVClass, TaskReporter reporter,
+ public TaskContext(JobConf conf, Class<?> iKClass, Class<?> iVClass,
+ Class<?> oKClass, Class<?> oVClass, TaskReporter reporter,
TaskAttemptID id) {
this.conf = conf;
this.iKClass = iKClass;
@@ -41,35 +44,35 @@ public class TaskContext {
this.taskAttemptID = id;
}
- public Class getInputKeyClass() {
+ public Class<?> getInputKeyClass() {
return iKClass;
}
- public void setInputKeyClass(Class klass) {
+ public void setInputKeyClass(Class<?> klass) {
this.iKClass = klass;
}
- public Class getInputValueClass() {
+ public Class<?> getInputValueClass() {
return iVClass;
}
- public void setInputValueClass(Class klass) {
+ public void setInputValueClass(Class<?> klass) {
this.iVClass = klass;
}
- public Class getOuputKeyClass() {
+ public Class<?> getOutputKeyClass() {
return this.oKClass;
}
- public void setOutputKeyClass(Class klass) {
+ public void setOutputKeyClass(Class<?> klass) {
this.oKClass = klass;
}
- public Class getOutputValueClass() {
+ public Class<?> getOutputValueClass() {
return this.oVClass;
}
- public void setOutputValueClass(Class klass) {
+ public void setOutputValueClass(Class<?> klass) {
this.oVClass = klass;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/BufferType.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/BufferType.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/BufferType.java
index a52be1f..bbdcd54 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/BufferType.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/BufferType.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.mapred.nativetask.buffer;
-public enum BufferType {
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
+public enum BufferType {
DIRECT_BUFFER,
-
HEAP_BUFFER
};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
index 24f402d..72c65f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
@@ -17,22 +17,17 @@
*/
package org.apache.hadoop.mapred.nativetask.buffer;
-import com.google.common.base.Charsets;
-
-import java.io.DataInput;
import java.io.EOFException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.PushbackInputStream;
-import java.io.UTFDataFormatException;
import java.nio.ByteBuffer;
+import org.apache.hadoop.classification.InterfaceAudience;
/**
* read data from a input buffer
*/
+@InterfaceAudience.Private
public class ByteBufferDataReader extends DataInputStream {
private ByteBuffer byteBuffer;
- private char lineCache[];
private java.io.DataInputStream javaReader;
public ByteBufferDataReader(InputBuffer buffer) {
@@ -130,6 +125,7 @@ public class ByteBufferDataReader extends DataInputStream {
return byteBuffer.getDouble();
}
+ @SuppressWarnings("deprecation")
@Override
public String readLine() throws IOException {
return javaReader.readLine();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
index aa87e0d..3d8f78b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
@@ -17,23 +17,21 @@
*/
package org.apache.hadoop.mapred.nativetask.buffer;
-import java.io.DataOutput;
import java.io.IOException;
-import java.io.UTFDataFormatException;
import java.nio.ByteBuffer;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.UnsignedInteger;
-import com.google.common.primitives.UnsignedInts;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
+import com.google.common.base.Preconditions;
+
/**
* DataOutputStream implementation which buffers data in a fixed-size
* ByteBuffer.
* When the byte buffer has filled up, synchronously passes the buffer
* to a downstream NativeDataTarget.
*/
+@InterfaceAudience.Private
public class ByteBufferDataWriter extends DataOutputStream {
private final ByteBuffer buffer;
private final NativeDataTarget target;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataInputStream.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataInputStream.java
index 0c1bc31..be17ef4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataInputStream.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataInputStream.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.mapred.nativetask.buffer;
import java.io.DataInput;
import java.io.InputStream;
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
public abstract class DataInputStream extends InputStream implements DataInput {
public abstract boolean hasUnReadData();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataOutputStream.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataOutputStream.java
index 6cd6782..38f9dcb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataOutputStream.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DataOutputStream.java
@@ -21,22 +21,19 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.OutputStream;
-public abstract class DataOutputStream extends OutputStream implements DataOutput {
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
+public abstract class DataOutputStream extends OutputStream implements DataOutput {
/**
* Check whether this buffer has enough space to store length of bytes
*
- * @param length
- * , length of bytes
- * @return
- * @throws IOException
+ * @param length length of bytes
*/
public abstract boolean shortOfSpace(int length) throws IOException;
/**
* Check whether there is unflushed data stored in the stream
- *
- * @return
*/
public abstract boolean hasUnFlushedData();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
index 82193b8..2120c6c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.mapred.nativetask.buffer;
import org.apache.hadoop.util.DirectBufferPool;
+import org.apache.hadoop.classification.InterfaceAudience;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+@InterfaceAudience.Private
public class InputBuffer implements Closeable {
static DirectBufferPool bufferPool = new DirectBufferPool();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/OutputBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/OutputBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/OutputBuffer.java
index 3c54948..8f33946 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/OutputBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/OutputBuffer.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.mapred.nativetask.buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
public class OutputBuffer {
protected ByteBuffer byteBuffer;
private final BufferType type;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
index 50d7816..1047bbf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred.nativetask.handlers;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.nativetask.Constants;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
@@ -30,9 +31,10 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
/**
* load data into a buffer signaled by a {@link BufferPuller}
*/
+@InterfaceAudience.Private
public class BufferPullee<IK, IV> implements IDataLoader {
- public static int KV_HEADER_LENGTH = Constants.SIZEOF_KV_LENGTH;
+ public static final int KV_HEADER_LENGTH = Constants.SIZEOF_KV_LENGTH;
private final SizedWritable<IK> tmpInputKey;
private final SizedWritable<IV> tmpInputValue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
index 704b664..18523ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
@@ -1,10 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.mapred.nativetask.handlers;
import java.io.IOException;
import java.nio.ByteBuffer;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.nativetask.Constants;
@@ -13,13 +32,12 @@ import org.apache.hadoop.mapred.nativetask.NativeDataSource;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
import org.apache.hadoop.mapred.nativetask.buffer.ByteBufferDataReader;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
-import org.apache.hadoop.mapred.nativetask.serde.SerializationFramework;
-import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.util.Progress;
/**
* actively signal a {@link BufferPullee} to load data into buffer and receive
*/
+@InterfaceAudience.Private
public class BufferPuller implements RawKeyValueIterator, DataReceiver {
private static Log LOG = LogFactory.getLog(BufferPuller.class);
@@ -108,8 +126,8 @@ public class BufferPuller implements RawKeyValueIterator, DataReceiver {
valueBytes = new byte[valueLength];
}
- nativeReader.read(keyBytes, 0, keyLength);
- nativeReader.read(valueBytes, 0, valueLength);
+ IOUtils.readFully(nativeReader, keyBytes, 0, keyLength);
+ IOUtils.readFully(nativeReader, valueBytes, 0, valueLength);
keyBuffer.reset(keyBytes, keyLength);
valueBuffer.reset(valueBytes, valueLength);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
index 8decad8..fe78325 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.nativetask.Constants;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
@@ -34,6 +35,7 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
/**
* collect data when signaled
*/
+@InterfaceAudience.Private
public class BufferPushee<OK, OV> implements Closeable {
private static Log LOG = LogFactory.getLog(BufferPushee.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
index 3713078..10d619b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
import org.apache.hadoop.mapred.nativetask.buffer.ByteBufferDataWriter;
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
/**
* actively push data into a buffer and signal a {@link BufferPushee} to collect it
*/
+@InterfaceAudience.Private
public class BufferPusher<K, V> implements OutputCollector<K, V> {
private static Log LOG = LogFactory.getLog(BufferPusher.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
index 6a57683..dad7af9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.mapred.nativetask.handlers;
-import static org.apache.hadoop.mapred.Task.Counter.COMBINE_INPUT_RECORDS;
-
import java.io.IOException;
import org.apache.commons.logging.Log;
@@ -37,13 +35,13 @@ import org.apache.hadoop.mapred.nativetask.TaskContext;
import org.apache.hadoop.mapred.nativetask.serde.SerializationFramework;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskCounter;
-public class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
-
- public static String NAME = "NativeTask.CombineHandler";
+class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
+ public static final String NAME = "NativeTask.CombineHandler";
private static Log LOG = LogFactory.getLog(NativeCollectorOnlyHandler.class);
- public static Command LOAD = new Command(1, "Load");
- public static Command COMBINE = new Command(4, "Combine");
+ public static final Command LOAD = new Command(1, "Load");
+ public static final Command COMBINE = new Command(4, "Combine");
public final CombinerRunner<K, V> combinerRunner;
private final INativeHandler nativeHandler;
@@ -66,13 +64,16 @@ public class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher
LOG.info("NativeTask Combiner is enabled, class = " + combinerClazz);
}
- final Counter combineInputCounter = context.getTaskReporter().getCounter(COMBINE_INPUT_RECORDS);
+ final Counter combineInputCounter = context.getTaskReporter().getCounter(
+ TaskCounter.COMBINE_INPUT_RECORDS);
final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(conf, context.getTaskAttemptId(),
combineInputCounter, context.getTaskReporter(), null);
final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, conf, DataChannel.INOUT);
- final BufferPusher<K, V> pusher = new BufferPusher<K, V>(context.getInputKeyClass(), context.getInputValueClass(),
+ @SuppressWarnings("unchecked")
+ final BufferPusher<K, V> pusher = new BufferPusher<K, V>((Class<K>)context.getInputKeyClass(),
+ (Class<V>)context.getInputValueClass(),
nativeHandler);
final BufferPuller puller = new BufferPuller(nativeHandler);
return new CombinerHandler<K, V>(nativeHandler, combinerRunner, puller, pusher);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
index ff472a6..d2cbf1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
@@ -20,9 +20,12 @@ package org.apache.hadoop.mapred.nativetask.handlers;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
/**
* an IDataLoader loads data on demand
*/
+@InterfaceAudience.Private
public interface IDataLoader {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
index 678e13d..284c3dc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskAttemptID;
@@ -41,14 +42,19 @@ import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
* Java Record Reader + Java Mapper + Native Collector
*/
@SuppressWarnings("unchecked")
+@InterfaceAudience.Private
public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Closeable {
- public static String NAME = "NativeTask.MCollectorOutputHandler";
+ public static final String NAME = "NativeTask.MCollectorOutputHandler";
private static Log LOG = LogFactory.getLog(NativeCollectorOnlyHandler.class);
- public static Command GET_OUTPUT_PATH = new Command(100, "GET_OUTPUT_PATH");
- public static Command GET_OUTPUT_INDEX_PATH = new Command(101, "GET_OUTPUT_INDEX_PATH");
- public static Command GET_SPILL_PATH = new Command(102, "GET_SPILL_PATH");
- public static Command GET_COMBINE_HANDLER = new Command(103, "GET_COMBINE_HANDLER");
+ public static final Command GET_OUTPUT_PATH =
+ new Command(100, "GET_OUTPUT_PATH");
+ public static final Command GET_OUTPUT_INDEX_PATH =
+ new Command(101, "GET_OUTPUT_INDEX_PATH");
+ public static final Command GET_SPILL_PATH =
+ new Command(102, "GET_SPILL_PATH");
+ public static final Command GET_COMBINE_HANDLER =
+ new Command(103, "GET_COMBINE_HANDLER");
private NativeTaskOutput output;
private int spillNumber = 0;
@@ -63,7 +69,7 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
ICombineHandler combinerHandler = null;
try {
final TaskContext combineContext = context.copyOf();
- combineContext.setInputKeyClass(context.getOuputKeyClass());
+ combineContext.setInputKeyClass(context.getOutputKeyClass());
combineContext.setInputValueClass(context.getOutputValueClass());
combinerHandler = CombinerHandler.create(combineContext);
@@ -76,7 +82,9 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
}
final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, context.getConf(), DataChannel.OUT);
- final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>(context.getOuputKeyClass(), context.getOutputValueClass(),
+ final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>(
+ (Class<K>)context.getOutputKeyClass(),
+ (Class<V>)context.getOutputValueClass(),
nativeHandler);
return new NativeCollectorOnlyHandler<K, V>(context, nativeHandler, kvPusher, combinerHandler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BoolWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BoolWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BoolWritableSerializer.java
index 9a026be..a4e8a56 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BoolWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BoolWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class BoolWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/ByteWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/ByteWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/ByteWritableSerializer.java
index 1ec2fdb..b029e67 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/ByteWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/ByteWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class ByteWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
index 2bd18d7..08264f9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
@@ -22,9 +22,11 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class BytesWritableSerializer implements INativeComparable, INativeSerializer<BytesWritable> {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DefaultSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DefaultSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DefaultSerializer.java
index d4fc7e0..fcef779 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DefaultSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DefaultSerializer.java
@@ -24,8 +24,12 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class DefaultSerializer implements INativeSerializer<Writable> {
static class ModifiedByteArrayOutputStream extends ByteArrayOutputStream {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DoubleWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DoubleWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DoubleWritableSerializer.java
index 8de0fba..a171f6f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DoubleWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/DoubleWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class DoubleWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/FloatWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/FloatWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/FloatWritableSerializer.java
index 4a2366c..bf2c959 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/FloatWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/FloatWritableSerializer.java
@@ -20,9 +20,12 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+
+@InterfaceAudience.Private
public class FloatWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
index 64c5810..8506d17 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.buffer.DataInputStream;
import org.apache.hadoop.mapred.nativetask.buffer.DataOutputStream;
import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
@@ -27,6 +28,7 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
/**
* serializes key-value pair
*/
+@InterfaceAudience.Private
public interface IKVSerializer {
/**
@@ -35,7 +37,7 @@ public interface IKVSerializer {
* @param value
* @throws IOException
*/
- public void updateLength(SizedWritable key, SizedWritable value) throws IOException;
+ public void updateLength(SizedWritable<?> key, SizedWritable<?> value) throws IOException;
/**
*
@@ -45,7 +47,8 @@ public interface IKVSerializer {
* @return bytes written
* @throws IOException
*/
- public int serializeKV(DataOutputStream out, SizedWritable key, SizedWritable value) throws IOException;
+ public int serializeKV(DataOutputStream out, SizedWritable<?> key,
+ SizedWritable<?> value) throws IOException;
/**
* serialize partitionId as well
@@ -56,7 +59,8 @@ public interface IKVSerializer {
* @return
* @throws IOException
*/
- public int serializePartitionKV(DataOutputStream out, int partitionId, SizedWritable key, SizedWritable value)
+ public int serializePartitionKV(DataOutputStream out, int partitionId,
+ SizedWritable<?> key, SizedWritable<?> value)
throws IOException;
/**
@@ -67,5 +71,5 @@ public interface IKVSerializer {
* @return bytes read
* @throws IOException
*/
- public int deserializeKV(DataInputStream in, SizedWritable key, SizedWritable value) throws IOException;
+ public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/INativeSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/INativeSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/INativeSerializer.java
index f61d12d..c970394 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/INativeSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/INativeSerializer.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
/**
* an INativeSerializer serializes and deserializes data transferred between
@@ -30,6 +32,8 @@ import java.io.IOException;
* you have to make sure the native side can serialize it correctly.
*
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public interface INativeSerializer<T> {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IntWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IntWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IntWritableSerializer.java
index e7e19a9..e79d182 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IntWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IntWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class IntWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
index 4b76df4..d650a4d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
@@ -22,18 +22,21 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.Constants;
import org.apache.hadoop.mapred.nativetask.buffer.DataInputStream;
import org.apache.hadoop.mapred.nativetask.buffer.DataOutputStream;
import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
-public class KVSerializer<K, V> implements IKVSerializer {
+@InterfaceAudience.Private
+public class KVSerializer<K, V> implements IKVSerializer {
+
private static final Log LOG = LogFactory.getLog(KVSerializer.class);
- public static int KV_HEAD_LENGTH = Constants.SIZEOF_KV_LENGTH;
+ public static final int KV_HEAD_LENGTH = Constants.SIZEOF_KV_LENGTH;
private final INativeSerializer<Writable> keySerializer;
private final INativeSerializer<Writable> valueSerializer;
@@ -45,19 +48,20 @@ public class KVSerializer<K, V> implements IKVSerializer {
}
@Override
- public void updateLength(SizedWritable key, SizedWritable value) throws IOException {
+ public void updateLength(SizedWritable<?> key, SizedWritable<?> value) throws IOException {
key.length = keySerializer.getLength(key.v);
value.length = valueSerializer.getLength(value.v);
return;
}
@Override
- public int serializeKV(DataOutputStream out, SizedWritable key, SizedWritable value) throws IOException {
+ public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) throws IOException {
return serializePartitionKV(out, -1, key, value);
}
@Override
- public int serializePartitionKV(DataOutputStream out, int partitionId, SizedWritable key, SizedWritable value)
+ public int serializePartitionKV(DataOutputStream out, int partitionId,
+ SizedWritable<?> key, SizedWritable<?> value)
throws IOException {
if (key.length == SizedWritable.INVALID_LENGTH || value.length == SizedWritable.INVALID_LENGTH) {
@@ -90,7 +94,8 @@ public class KVSerializer<K, V> implements IKVSerializer {
}
@Override
- public int deserializeKV(DataInputStream in, SizedWritable key, SizedWritable value) throws IOException {
+ public int deserializeKV(DataInputStream in, SizedWritable<?> key,
+ SizedWritable<?> value) throws IOException {
if (!in.hasUnReadData()) {
return 0;
[17/50] [abbrv] git commit: MAPREDUCE-5976. native-task: should not
fail to build if snappy is missing (Manu Zhang)
Posted by ar...@apache.org.
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616115 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/432f641b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/432f641b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/432f641b
Branch: refs/heads/HDFS-6581
Commit: 432f641bc2f6c7fdb67da578233cbe07fdfbf1ea
Parents: 7ecaa81
Author: Sean Zhong <se...@apache.org>
Authored: Wed Aug 6 07:32:49 2014 +0000
Committer: Sean Zhong <se...@apache.org>
Committed: Wed Aug 6 07:32:49 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../src/CMakeLists.txt | 8 +-
.../NativeMapOutputCollectorDelegator.java | 34 ++--
.../hadoop/mapred/nativetask/NativeRuntime.java | 11 +-
.../mapred/nativetask/util/SnappyUtil.java | 32 ----
.../src/main/native/src/codec/SnappyCodec.cc | 9 +-
.../src/main/native/src/codec/snappy-c.h | 138 ----------------
.../src/main/native/src/codec/snappy.h | 163 -------------------
.../src/main/native/src/lib/Compressions.cc | 9 +
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 25 +++
...che_hadoop_mapred_nativetask_NativeRuntime.h | 8 +
.../src/main/native/test/TestCompressions.cc | 69 ++++----
.../src/main/native/test/TestIFile.cc | 3 +
.../src/main/native/test/lib/TestReadBuffer.cc | 4 +-
14 files changed, 117 insertions(+), 397 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 9dddcd5..4b77262 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -10,3 +10,4 @@ MAPREDUCE-5991. native-task should not run unit tests if native profile is not e
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
+MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 36dbd9c..f38021d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -122,7 +122,6 @@ CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET)
CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
@@ -139,6 +138,7 @@ if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
set(SNAPPY_SOURCE_FILES
"${D}/src/codec/SnappyCodec.cc")
else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ set(SNAPPY_LIBRARY "")
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
IF(REQUIRE_SNAPPY)
@@ -146,6 +146,8 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
ENDIF(REQUIRE_SNAPPY)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+
include_directories(
${GENERATED_JAVAH}
${D}
@@ -154,15 +156,11 @@ include_directories(
${D}/src/lib
${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
- #${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
-#SET(CMAKE_SOURCE_DIR "/cygdrive/c/Users/tianlunz/repo/hadoop-2.2.0-src/hadoop-common-project/hadoop-common/src")
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index a48b5c3..beb3851 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.File;
import java.io.IOException;
+import com.google.common.base.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -80,29 +81,22 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
Class comparatorClass = job.getClass(MRJobConfig.KEY_COMPARATOR, null, RawComparator.class);
if (comparatorClass != null && !Platforms.define(comparatorClass)) {
- String message = "Native output collector don't support customized java comparator "
+ String message = "Native output collector doesn't support customized java comparator "
+ job.get(MRJobConfig.KEY_COMPARATOR);
LOG.error(message);
throw new InvalidJobConfException(message);
}
- if (job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false) == true) {
- if (!isCodecSupported(job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC))) {
- String message = "Native output collector don't support compression codec "
- + job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC) + ", We support Gzip, Lz4, snappy";
- LOG.error(message);
- throw new InvalidJobConfException(message);
- }
- }
+
if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) {
- String message = "Native-Task don't support sort class " + job.get(Constants.MAP_SORT_CLASS);
+ String message = "Native-Task doesn't support sort class " + job.get(Constants.MAP_SORT_CLASS);
LOG.error(message);
throw new InvalidJobConfException(message);
}
if (job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, false) == true) {
- String message = "Native-Task don't support secure shuffle";
+ String message = "Native-Task doesn't support secure shuffle";
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -116,7 +110,7 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
LOG.error(message);
throw new InvalidJobConfException(message);
} else if (!Platforms.support(keyCls.getName(), serializer, job)) {
- String message = "Native output collector don't support this key, this key is not comparable in native "
+ String message = "Native output collector doesn't support this key, this key is not comparable in native "
+ keyCls.getName();
LOG.error(message);
throw new InvalidJobConfException(message);
@@ -129,6 +123,14 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
final boolean ret = NativeRuntime.isNativeLibraryLoaded();
if (ret) {
+ if (job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false)) {
+ String codec = job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC);
+ if (!NativeRuntime.supportsCompressionCodec(codec.getBytes(Charsets.UTF_8))) {
+ String message = "Native output collector doesn't support compression codec " + codec;
+ LOG.error(message);
+ throw new InvalidJobConfException(message);
+ }
+ }
NativeRuntime.configure(job);
final long updateInterval = job.getLong(Constants.NATIVE_STATUS_UPDATE_INTERVAL,
@@ -159,12 +161,4 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
LOG.info("Native output collector can be successfully enabled!");
}
- private boolean isCodecSupported(String string) {
- if ("org.apache.hadoop.io.compress.SnappyCodec".equals(string)
- || "org.apache.hadoop.io.compress.GzipCodec".equals(string)
- || "org.apache.hadoop.io.compress.Lz4Codec".equals(string)) {
- return true;
- }
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index 65ce652..5ce3c89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Task.TaskReporter;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
-import org.apache.hadoop.mapred.nativetask.util.SnappyUtil;
import org.apache.hadoop.util.VersionInfo;
/**
@@ -48,11 +47,6 @@ public class NativeRuntime {
static {
try {
- if (false == SnappyUtil.isNativeSnappyLoaded(conf)) {
- throw new IOException("Snappy library cannot be loaded");
- } else {
- LOG.info("Snappy native library is available");
- }
System.loadLibrary("nativetask");
LOG.info("Nativetask JNI library loaded.");
nativeLibraryLoaded = true;
@@ -162,6 +156,11 @@ public class NativeRuntime {
********************************************************/
/**
+ * Check whether the native side has compression codec support built in
+ */
+ public native static boolean supportsCompressionCodec(byte[] codec);
+
+ /**
* Config the native runtime with mapreduce job configurations.
*
* @param configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SnappyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SnappyUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SnappyUtil.java
deleted file mode 100644
index fb15960..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SnappyUtil.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.compress.SnappyCodec;
-
-public class SnappyUtil {
-
- public static boolean isNativeSnappyLoaded(Configuration conf) {
- return SnappyCodec.isNativeCodeLoaded() && conf.getBoolean(
- CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
- CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
index bf2cde2..0504033 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
@@ -16,11 +16,15 @@
* limitations under the License.
*/
-#include "snappy-c.h"
+#include "config.h"
+
+#if defined HADOOP_SNAPPY_LIBRARY
#include "commons.h"
#include "NativeTask.h"
#include "SnappyCodec.h"
+#include <snappy-c.h>
+
namespace NativeTask {
SnappyCompressStream::SnappyCompressStream(OutputStream * stream, uint32_t bufferSizeHint)
@@ -89,5 +93,6 @@ uint32_t SnappyDecompressStream::decompressOneBlock(uint32_t compressedSize, voi
uint64_t SnappyDecompressStream::maxCompressedLength(uint64_t origLength) {
return snappy_max_compressed_length(origLength);
}
-
} // namespace NativeTask
+
+#endif // define HADOOP_SNAPPY_LIBRARY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy-c.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy-c.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy-c.h
deleted file mode 100644
index 4ccc742..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy-c.h
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2011 Martin Gieseking <ma...@uos.de>.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Plain C interface (a wrapper around the C++ implementation).
- */
-
-#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
-#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stddef.h>
-
-/*
- * Return values; see the documentation for each function to know
- * what each can return.
- */
-typedef enum {
- SNAPPY_OK = 0,
- SNAPPY_INVALID_INPUT = 1,
- SNAPPY_BUFFER_TOO_SMALL = 2
-} snappy_status;
-
-/*
- * Takes the data stored in "input[0..input_length-1]" and stores
- * it in the array pointed to by "compressed".
- *
- * <compressed_length> signals the space available in "compressed".
- * If it is not at least equal to "snappy_max_compressed_length(input_length)",
- * SNAPPY_BUFFER_TOO_SMALL is returned. After successful compression,
- * <compressed_length> contains the true length of the compressed output,
- * and SNAPPY_OK is returned.
- *
- * Example:
- * size_t output_length = snappy_max_compressed_length(input_length);
- * char* output = (char*)malloc(output_length);
- * if (snappy_compress(input, input_length, output, &output_length)
- * == SNAPPY_OK) {
- * ... Process(output, output_length) ...
- * }
- * free(output);
- */
-snappy_status snappy_compress(const char* input,
- size_t input_length,
- char* compressed,
- size_t* compressed_length);
-
-/*
- * Given data in "compressed[0..compressed_length-1]" generated by
- * calling the snappy_compress routine, this routine stores
- * the uncompressed data to
- * uncompressed[0..uncompressed_length-1].
- * Returns failure (a value not equal to SNAPPY_OK) if the message
- * is corrupted and could not be decrypted.
- *
- * <uncompressed_length> signals the space available in "uncompressed".
- * If it is not at least equal to the value returned by
- * snappy_uncompressed_length for this stream, SNAPPY_BUFFER_TOO_SMALL
- * is returned. After successful decompression, <uncompressed_length>
- * contains the true length of the decompressed output.
- *
- * Example:
- * size_t output_length;
- * if (snappy_uncompressed_length(input, input_length, &output_length)
- * != SNAPPY_OK) {
- * ... fail ...
- * }
- * char* output = (char*)malloc(output_length);
- * if (snappy_uncompress(input, input_length, output, &output_length)
- * == SNAPPY_OK) {
- * ... Process(output, output_length) ...
- * }
- * free(output);
- */
-snappy_status snappy_uncompress(const char* compressed,
- size_t compressed_length,
- char* uncompressed,
- size_t* uncompressed_length);
-
-/*
- * Returns the maximal size of the compressed representation of
- * input data that is "source_length" bytes in length.
- */
-size_t snappy_max_compressed_length(size_t source_length);
-
-/*
- * REQUIRES: "compressed[]" was produced by snappy_compress()
- * Returns SNAPPY_OK and stores the length of the uncompressed data in
- * *result normally. Returns SNAPPY_INVALID_INPUT on parsing error.
- * This operation takes O(1) time.
- */
-snappy_status snappy_uncompressed_length(const char* compressed,
- size_t compressed_length,
- size_t* result);
-
-/*
- * Check if the contents of "compressed[]" can be uncompressed successfully.
- * Does not return the uncompressed data; if so, returns SNAPPY_OK,
- * or if not, returns SNAPPY_INVALID_INPUT.
- * Takes time proportional to compressed_length, but is usually at least a
- * factor of four faster than actual decompression.
- */
-snappy_status snappy_validate_compressed_buffer(const char* compressed,
- size_t compressed_length);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif
-
-#endif /* UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy.h
deleted file mode 100644
index 03ef6ce..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy.h
+++ /dev/null
@@ -1,163 +0,0 @@
-// Copyright 2005 and onwards Google Inc.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// A light-weight compression algorithm. It is designed for speed of
-// compression and decompression, rather than for the utmost in space
-// savings.
-//
-// For getting better compression ratios when you are compressing data
-// with long repeated sequences or compressing data that is similar to
-// other data, while still compressing fast, you might look at first
-// using BMDiff and then compressing the output of BMDiff with
-// Snappy.
-
-#ifndef UTIL_SNAPPY_SNAPPY_H__
-#define UTIL_SNAPPY_SNAPPY_H__
-
-#include <stddef.h>
-#include <string>
-
-#include "snappy-stubs-public.h"
-
-namespace snappy {
- class Source;
- class Sink;
-
- // ------------------------------------------------------------------------
- // Generic compression/decompression routines.
- // ------------------------------------------------------------------------
-
- // Compress the bytes read from "*source" and append to "*sink". Return the
- // number of bytes written.
- size_t Compress(Source* source, Sink* sink);
-
- // Find the uncompressed length of the given stream, as given by the header.
- // Note that the true length could deviate from this; the stream could e.g.
- // be truncated.
- //
- // Also note that this leaves "*source" in a state that is unsuitable for
- // further operations, such as RawUncompress(). You will need to rewind
- // or recreate the source yourself before attempting any further calls.
- bool GetUncompressedLength(Source* source, uint32* result);
-
- // ------------------------------------------------------------------------
- // Higher-level string based routines (should be sufficient for most users)
- // ------------------------------------------------------------------------
-
- // Sets "*output" to the compressed version of "input[0,input_length-1]".
- // Original contents of *output are lost.
- //
- // REQUIRES: "input[]" is not an alias of "*output".
- size_t Compress(const char* input, size_t input_length, string* output);
-
- // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed".
- // Original contents of "*uncompressed" are lost.
- //
- // REQUIRES: "compressed[]" is not an alias of "*uncompressed".
- //
- // returns false if the message is corrupted and could not be decompressed
- bool Uncompress(const char* compressed, size_t compressed_length,
- string* uncompressed);
-
-
- // ------------------------------------------------------------------------
- // Lower-level character array based routines. May be useful for
- // efficiency reasons in certain circumstances.
- // ------------------------------------------------------------------------
-
- // REQUIRES: "compressed" must point to an area of memory that is at
- // least "MaxCompressedLength(input_length)" bytes in length.
- //
- // Takes the data stored in "input[0..input_length]" and stores
- // it in the array pointed to by "compressed".
- //
- // "*compressed_length" is set to the length of the compressed output.
- //
- // Example:
- // char* output = new char[snappy::MaxCompressedLength(input_length)];
- // size_t output_length;
- // RawCompress(input, input_length, output, &output_length);
- // ... Process(output, output_length) ...
- // delete [] output;
- void RawCompress(const char* input,
- size_t input_length,
- char* compressed,
- size_t* compressed_length);
-
- // Given data in "compressed[0..compressed_length-1]" generated by
- // calling the Snappy::Compress routine, this routine
- // stores the uncompressed data to
- // uncompressed[0..GetUncompressedLength(compressed)-1]
- // returns false if the message is corrupted and could not be decrypted
- bool RawUncompress(const char* compressed, size_t compressed_length,
- char* uncompressed);
-
- // Given data from the byte source 'compressed' generated by calling
- // the Snappy::Compress routine, this routine stores the uncompressed
- // data to
- // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1]
- // returns false if the message is corrupted and could not be decrypted
- bool RawUncompress(Source* compressed, char* uncompressed);
-
- // Returns the maximal size of the compressed representation of
- // input data that is "source_bytes" bytes in length;
- size_t MaxCompressedLength(size_t source_bytes);
-
- // REQUIRES: "compressed[]" was produced by RawCompress() or Compress()
- // Returns true and stores the length of the uncompressed data in
- // *result normally. Returns false on parsing error.
- // This operation takes O(1) time.
- bool GetUncompressedLength(const char* compressed, size_t compressed_length,
- size_t* result);
-
- // Returns true iff the contents of "compressed[]" can be uncompressed
- // successfully. Does not return the uncompressed data. Takes
- // time proportional to compressed_length, but is usually at least
- // a factor of four faster than actual decompression.
- bool IsValidCompressedBuffer(const char* compressed,
- size_t compressed_length);
-
- // The size of a compression block. Note that many parts of the compression
- // code assumes that kBlockSize <= 65536; in particular, the hash table
- // can only store 16-bit offsets, and EmitCopy() also assumes the offset
- // is 65535 bytes or less. Note also that if you change this, it will
- // affect the framing format (see framing_format.txt).
- //
- // Note that there might be older data around that is compressed with larger
- // block sizes, so the decompression code should not rely on the
- // non-existence of long backreferences.
- static const int kBlockLog = 16;
- static const size_t kBlockSize = 1 << kBlockLog;
-
- static const int kMaxHashTableBits = 14;
- static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
-
-} // end namespace snappy
-
-
-#endif // UTIL_SNAPPY_SNAPPY_H__
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
index e2ce730..0f954e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
@@ -17,6 +17,7 @@
*/
#include "commons.h"
+#include "config.h"
#include "SyncUtils.h"
#include "Compressions.h"
#include "codec/GzipCodec.h"
@@ -110,7 +111,11 @@ CompressStream * Compressions::getCompressionStream(const string & codec, Output
return new GzipCompressStream(stream, bufferSizeHint);
}
if (codec == SnappyCodec.name) {
+#if defined HADOOP_SNAPPY_LIBRARY
return new SnappyCompressStream(stream, bufferSizeHint);
+#else
+ THROW_EXCEPTION(UnsupportException, "Snappy library is not loaded");
+#endif
}
if (codec == Lz4Codec.name) {
return new Lz4CompressStream(stream, bufferSizeHint);
@@ -124,7 +129,11 @@ DecompressStream * Compressions::getDecompressionStream(const string & codec, In
return new GzipDecompressStream(stream, bufferSizeHint);
}
if (codec == SnappyCodec.name) {
+#if defined HADOOP_SNAPPY_LIBRARY
return new SnappyDecompressStream(stream, bufferSizeHint);
+#else
+ THROW_EXCEPTION(UnsupportException, "Snappy library is not loaded");
+#endif
}
if (codec == Lz4Codec.name) {
return new Lz4DecompressStream(stream, bufferSizeHint);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
index 3e81565..28aeaf3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
@@ -19,6 +19,7 @@
#ifndef QUICK_BUILD
#include "org_apache_hadoop_mapred_nativetask_NativeRuntime.h"
#endif
+#include "config.h"
#include "commons.h"
#include "jniutils.h"
#include "NativeObjectFactory.h"
@@ -31,6 +32,30 @@ using namespace NativeTask;
/*
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
+ * Method: supportCompressionCodec
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
+ (JNIEnv *jenv, jclass clazz, jbyteArray codec)
+{
+ const std::string codecString = JNU_ByteArrayToString(jenv, codec);
+ if ("org.apache.hadoop.io.compress.GzipCodec" == codecString) {
+ return JNI_TRUE;
+ } else if ("org.apache.hadoop.io.compress.Lz4Codec" == codecString) {
+ return JNI_TRUE;
+ } else if ("org.apache.hadoop.io.compress.SnappyCodec" == codecString) {
+#if defined HADOOP_SNAPPY_LIBRARY
+ return JNI_TRUE;
+#else
+ return JNI_FALSE;
+#endif
+ } else {
+ return JNI_FALSE;
+ }
+}
+
+/*
+ * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
*/JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
index 6d780cf..915c532 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
@@ -9,6 +9,14 @@ extern "C" {
#endif
/*
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
+ * Method: supportCompressionCodec
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
+ (JNIEnv *, jclass, jbyteArray);
+
+/*
+ * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
*/JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(JNIEnv *,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
index 68e8368..de02ac7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
@@ -17,7 +17,7 @@
*/
#include "lz4.h"
-#include "snappy.h"
+#include "config.h"
#include "commons.h"
#include "Path.h"
#include "BufferStream.h"
@@ -25,9 +25,29 @@
#include "Compressions.h"
#include "test_commons.h"
-void TestCodec(const string & codec, const string & data, char * buff, char * buff2, size_t buffLen,
- uint32_t buffhint) {
+#if defined HADOOP_SNAPPY_LIBRARY
+#include <snappy.h>
+#endif
+
+void TestCodec(const string & codec) {
+ string data;
+ size_t length = TestConfig.getInt("compression.input.length", 100 * 1024 * 1024);
+ uint32_t buffhint = TestConfig.getInt("compression.buffer.hint", 128 * 1024);
+ string type = TestConfig.get("compression.input.type", "bytes");
Timer timer;
+ GenerateKVTextLength(data, length, type);
+ LOG("%s", timer.getInterval("Generate data").c_str());
+
+ InputBuffer inputBuffer = InputBuffer(data);
+ size_t buffLen = data.length() / 2 * 3;
+
+ timer.reset();
+ char * buff = new char[buffLen];
+ char * buff2 = new char[buffLen];
+ memset(buff, 0, buffLen);
+ memset(buff2, 0, buffLen);
+ LOG("%s", timer.getInterval("memset buffer to prevent missing page").c_str());
+
OutputBuffer outputBuffer = OutputBuffer(buff, buffLen);
CompressStream * compressor = Compressions::getCompressionStream(codec, &outputBuffer, buffhint);
@@ -57,35 +77,10 @@ void TestCodec(const string & codec, const string & data, char * buff, char * bu
ASSERT_EQ(data.length(), total);
ASSERT_EQ(0, memcmp(data.c_str(), buff2, total));
- delete compressor;
- delete decompressor;
-}
-
-TEST(Perf, Compressions) {
- string data;
- size_t length = TestConfig.getInt("compression.input.length", 100 * 1024 * 1024);
- uint32_t buffhint = TestConfig.getInt("compression.buffer.hint", 128 * 1024);
- string type = TestConfig.get("compression.input.type", "bytes");
- Timer timer;
- GenerateKVTextLength(data, length, type);
- LOG("%s", timer.getInterval("Generate data").c_str());
-
- InputBuffer inputBuffer = InputBuffer(data);
- size_t buffLen = data.length() / 2 * 3;
-
- timer.reset();
- char * buff = new char[buffLen];
- char * buff2 = new char[buffLen];
- memset(buff, 0, buffLen);
- memset(buff2, 0, buffLen);
- LOG("%s", timer.getInterval("memset buffer to prevent missing page").c_str());
-
- TestCodec("org.apache.hadoop.io.compress.SnappyCodec", data, buff, buff2, buffLen, buffhint);
- TestCodec("org.apache.hadoop.io.compress.Lz4Codec", data, buff, buff2, buffLen, buffhint);
- TestCodec("org.apache.hadoop.io.compress.GzipCodec", data, buff, buff2, buffLen, buffhint);
-
delete[] buff;
delete[] buff2;
+ delete compressor;
+ delete decompressor;
}
TEST(Perf, CompressionUtil) {
@@ -173,6 +168,9 @@ public:
}
};
+TEST(Perf, GzipCodec) {
+ TestCodec("org.apache.hadoop.io.compress.GzipCodec");
+}
void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t blockSize,
int times) {
@@ -226,6 +224,12 @@ TEST(Perf, RawCompressionLz4) {
printf("%s - Total\n", total.toString().c_str());
}
+TEST(Perf, Lz4Codec) {
+ TestCodec("org.apache.hadoop.io.compress.Lz4Codec");
+}
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
void MeasureSingleFileSnappy(const string & path, CompressResult & total, size_t blockSize,
int times) {
string data;
@@ -278,3 +282,8 @@ TEST(Perf, RawCompressionSnappy) {
printf("%s - Total\n", total.toString().c_str());
}
+TEST(Perf, SnappyCodec) {
+ TestCodec("org.apache.hadoop.io.compress.SnappyCodec");
+}
+
+#endif // define HADOOP_SNAPPY_LIBRARY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 22885b2..1d8b520 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -18,6 +18,7 @@
#include <algorithm>
#include "commons.h"
+#include "config.h"
#include "BufferStream.h"
#include "FileSystem.h"
#include "IFile.h"
@@ -90,7 +91,9 @@ TEST(IFile, WriteRead) {
TestIFileReadWrite(TextType, partition, size, kvs);
TestIFileReadWrite(BytesType, partition, size, kvs);
TestIFileReadWrite(UnknownType, partition, size, kvs);
+#if defined HADOOP_SNAPPY_LIBRARY
TestIFileReadWrite(TextType, partition, size, kvs, "org.apache.hadoop.io.compress.SnappyCodec");
+#endif
}
void TestIFileWriteRead2(vector<pair<string, string> > & kvs, char * buff, size_t buffsize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
index 648fe80..cbb636f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
@@ -17,6 +17,7 @@
*/
#include "commons.h"
+#include "config.h"
#include "BufferStream.h"
#include "Buffers.h"
#include "test_commons.h"
@@ -43,6 +44,7 @@ TEST(Buffers, AppendRead) {
}
}
+#if defined HADOOP_SNAPPY_LIBRARY
TEST(Buffers, AppendReadSnappy) {
string codec = "org.apache.hadoop.io.compress.SnappyCodec";
vector<string> data;
@@ -64,4 +66,4 @@ TEST(Buffers, AppendReadSnappy) {
ASSERT_EQ(data[i], string(rd, data[i].length()));
}
}
-
+#endif
[36/50] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of
unused code. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
deleted file mode 100644
index eec8091..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Hash.h"
-
-namespace NativeTask {
-
-} // namespace NativeTask
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
deleted file mode 100644
index 4f99d26..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef HASH_H_
-#define HASH_H_
-
-#include <stdint.h>
-#include <stdlib.h>
-
-// Hash function for a byte array.
-extern uint64_t CityHash64(const char *buf, size_t len);
-
-// Hash function for a byte array. For convenience, a 64-bit seed is also
-// hashed into the result.
-extern uint64_t CityHash64WithSeed(const char *buf, size_t len, uint64_t seed);
-
-namespace NativeTask {
-
-class Hash {
-public:
- /**
- * Compatible with hadoop Text & BytesWritable hash
- */
- inline static int32_t BytesHash(const char * bytes, uint32_t length) {
- int32_t hash = 1;
- for (uint32_t i = 0; i < length; i++)
- hash = (31 * hash) + (int32_t)bytes[i];
- return hash;
- }
-
- /**
- * Unsigned version of BytesHash
- */
- inline static uint32_t BytesHashU(const char * bytes, uint32_t length) {
- uint32_t hash = 1;
- for (uint32_t i = 0; i < length; i++)
- hash = (31U * hash) + (uint32_t)bytes[i];
- return hash;
- }
-
- /**
- * City hash, faster for longer input
- */
- inline static uint64_t CityHash(const char * bytes, uint32_t length) {
- return CityHash64(bytes, length);
- }
-
- /**
- * City hash, faster for longer input
- */
- inline static uint64_t CityHashWithSeed(const char * bytes, uint32_t length, uint64_t seed) {
- return CityHash64WithSeed(bytes, length, seed);
- }
-};
-
-} // namespace NativeTask
-
-#endif /* HASH_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
index e6cfec2..6266b1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
@@ -17,8 +17,8 @@
*/
#include <math.h>
-#include "commons.h"
-#include "Random.h"
+#include "lib/commons.h"
+#include "util/Random.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
index 7baaa0c..3ed7435 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
@@ -17,8 +17,8 @@
*/
#include <stdarg.h>
-#include "commons.h"
-#include "StringUtil.h"
+#include "lib/commons.h"
+#include "util/StringUtil.h"
namespace NativeTask {
@@ -201,8 +201,9 @@ bool StringUtil::StartsWith(const string & str, const string & prefix) {
}
bool StringUtil::EndsWith(const string & str, const string & suffix) {
- if ((suffix.length() > str.length())
- || (memcmp(str.data() + str.length() - suffix.length(), suffix.data(), suffix.length()) != 0)) {
+ if ((suffix.length() > str.length()) ||
+ (memcmp(str.data() + str.length() - suffix.length(),
+ suffix.data(), suffix.length()) != 0)) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc
index c3fb307..357a68d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "jniutils.h"
-#include "StringUtil.h"
-#include "SyncUtils.h"
+#include "lib/commons.h"
+#include "lib/jniutils.h"
+#include "util/StringUtil.h"
+#include "util/SyncUtils.h"
namespace NativeTask {
@@ -52,110 +52,4 @@ void Lock::unlock() {
PthreadCall("unlock", pthread_mutex_unlock(&_mutex));
}
-#ifdef __MACH__
-SpinLock::SpinLock() : _spin(0) {
-}
-
-SpinLock::~SpinLock() {
-
-}
-
-void SpinLock::lock() {
- OSSpinLockLock(&_spin);
-}
-
-void SpinLock::unlock() {
- OSSpinLockUnlock(&_spin);
-}
-#else
-SpinLock::SpinLock() {
- PthreadCall("init mutex", pthread_spin_init(&_spin, 0));
-}
-
-SpinLock::~SpinLock() {
- PthreadCall("destroy mutex", pthread_spin_destroy(&_spin));
-}
-
-void SpinLock::lock() {
- PthreadCall("lock", pthread_spin_lock(&_spin));
-}
-
-void SpinLock::unlock() {
- PthreadCall("unlock", pthread_spin_unlock(&_spin));
-}
-#endif
-
-Condition::Condition(Lock* mu)
- : _lock(mu) {
- PthreadCall("init cv", pthread_cond_init(&_condition, NULL));
-}
-
-Condition::~Condition() {
- PthreadCall("destroy cv", pthread_cond_destroy(&_condition));
-}
-
-void Condition::wait() {
- PthreadCall("wait", pthread_cond_wait(&_condition, &_lock->_mutex));
-}
-
-void Condition::signal() {
- PthreadCall("signal", pthread_cond_signal(&_condition));
-}
-
-void Condition::signalAll() {
- PthreadCall("broadcast", pthread_cond_broadcast(&_condition));
-}
-
-void * Thread::ThreadRunner(void * pthis) {
- try {
- ((Thread*)pthis)->run();
- } catch (std::exception & e) {
- LOG("err!!!! %s", e.what());
- }
- return NULL;
-}
-
-Thread::Thread()
- : _thread((pthread_t)0), // safe for linux & macos
- _runable(NULL) {
-}
-
-Thread::Thread(Runnable * runnable)
- : _thread((pthread_t)0), _runable(runnable) {
-}
-
-void Thread::setTask(const Runnable & runnable) {
- _runable = const_cast<Runnable*>(&runnable);
-}
-
-Thread::~Thread() {
-
-}
-
-void Thread::start() {
- PthreadCall("pthread_create", pthread_create(&_thread, NULL, ThreadRunner, this));
-}
-
-void Thread::join() {
- PthreadCall("pthread_join", pthread_join(_thread, NULL));
-}
-
-void Thread::stop() {
- PthreadCall("pthread_cancel", pthread_cancel(_thread));
-}
-
-void Thread::run() {
- if (_runable != NULL) {
- _runable->run();
- }
-}
-
-void Thread::EnableJNI() {
- JNU_AttachCurrentThread();
-}
-
-void Thread::ReleaseJNI() {
- JNU_DetachCurrentThread();
-}
-
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h
index 15ce2c8..270f851 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h
@@ -47,38 +47,6 @@ private:
void operator=(const Lock&);
};
-class SpinLock {
-public:
- SpinLock();
- ~SpinLock();
-
- void lock();
- void unlock();
-
-private:
-#ifdef __MACH__
- OSSpinLock _spin;
-#else
- pthread_spinlock_t _spin;
-#endif
-
- // No copying
- SpinLock(const Lock&);
- void operator=(const Lock&);
-};
-
-class Condition {
-public:
- explicit Condition(Lock* mu);
- ~Condition();
- void wait();
- void signal();
- void signalAll();
-private:
- pthread_cond_t _condition;
- Lock* _lock;
-};
-
template<typename LockT>
class ScopeLock {
public:
@@ -97,202 +65,6 @@ private:
void operator=(const ScopeLock&);
};
-class Runnable {
-public:
- virtual ~Runnable() {
- }
- virtual void run() = 0;
-};
-
-class Thread : public Runnable {
-protected:
- pthread_t _thread;
- Runnable * _runable;
-public:
- Thread();
- Thread(Runnable * runnable);
- virtual ~Thread();
-
- void setTask(const Runnable & runnable);
- void start();
- void join();
- void stop();
- virtual void run();
-
- /**
- * Enable JNI for current thread
- */
- static void EnableJNI();
- /**
- * Release JNI when thread is at end if current
- * thread called EnableJNI before
- */
- static void ReleaseJNI();
-private:
- static void * ThreadRunner(void * pthis);
-};
-
-// Sure <tr1/functional> is better
-template<typename Subject, typename Method>
-class FunctionRunner : public Runnable {
-protected:
- Subject & _subject;
- Method _method;
-public:
- FunctionRunner(Subject & subject, Method method)
- : _subject(subject), _method(method) {
- }
-
- virtual void run() {
- (_subject.*_method)();
- }
-};
-
-template<typename Subject, typename Method, typename Arg>
-class FunctionRunner1 : public Runnable {
-protected:
- Subject & _subject;
- Method _method;
- Arg _arg;
-public:
- FunctionRunner1(Subject & subject, Method method, Arg arg)
- : _subject(subject), _method(method), _arg(arg) {
- }
-
- virtual void run() {
- (_subject.*_method)(_arg);
- }
-};
-
-template<typename Subject, typename Method, typename Arg1, typename Arg2>
-class FunctionRunner2 : public Runnable {
-protected:
- Subject & _subject;
- Method _method;
- Arg1 _arg1;
- Arg2 _arg2;
-public:
- FunctionRunner2(Subject & subject, Method method, Arg1 arg1, Arg2 arg2)
- : _subject(subject), _method(method), _arg1(arg1), _arg2(arg2) {
- }
-
- virtual void run() {
- (_subject.*_method)(_arg1, _arg2);
- }
-};
-
-template<typename Subject, typename Method>
-inline FunctionRunner<Subject, Method> * BindNew(Subject & subject, Method method) {
- return new FunctionRunner<Subject, Method>(subject, method);
-}
-
-template<typename Subject, typename Method, typename Arg>
-inline FunctionRunner1<Subject, Method, Arg> * BindNew(Subject & subject, Method method, Arg arg) {
- return new FunctionRunner1<Subject, Method, Arg>(subject, method, arg);
-}
-
-template<typename Subject, typename Method, typename Arg1, typename Arg2>
-inline FunctionRunner2<Subject, Method, Arg1, Arg2> * BindNew(Subject & subject, Method method,
- Arg1 arg1, Arg2 arg2) {
- return new FunctionRunner2<Subject, Method, Arg1, Arg2>(subject, method, arg1, arg2);
-}
-
-class ConcurrentIndex {
-private:
- size_t _index;
- size_t _end;
- SpinLock _lock;
-public:
- ConcurrentIndex(size_t count)
- : _index(0), _end(count) {
- }
-
- ConcurrentIndex(size_t start, size_t end)
- : _index(start), _end(end) {
- }
-
- size_t count() {
- return _end;
- }
-
- ssize_t next() {
- ScopeLock<SpinLock> autoLock(_lock);
- if (_index == _end) {
- return -1;
- } else {
- ssize_t ret = _index;
- _index++;
- return ret;
- }
- }
-};
-
-template<typename Subject, typename Method, typename RangeType>
-class ParallelForWorker : public Runnable {
-protected:
- ConcurrentIndex * _index;
- Subject * _subject;
- Method _method;
-public:
- ParallelForWorker()
- : _index(NULL), _subject(NULL) {
- }
-
- ParallelForWorker(ConcurrentIndex * index, Subject * subject, Method method)
- : _index(index), _subject(subject), _method(method) {
- }
-
- void reset(ConcurrentIndex * index, Subject * subject, Method method) {
- _index = index;
- _subject = subject;
- _method = method;
- }
-
- virtual void run() {
- ssize_t i;
- while ((i = _index->next()) >= 0) {
- (_subject->*_method)(i);
- }
- }
-};
-
-template<typename Subject, typename Method, typename RangeType>
-void ParallelFor(Subject & subject, Method method, RangeType begin, RangeType end,
- size_t thread_num) {
- RangeType count = end - begin;
- if (thread_num <= 1 || count <= 1) {
- for (RangeType i = begin; i < end; i++) {
- (subject.*method)(i);
- }
- } else if (thread_num == 2) {
- ConcurrentIndex index = ConcurrentIndex(begin, end);
- ParallelForWorker<Subject, Method, RangeType> workers[2];
- Thread sideThread;
- workers[0].reset(&index, &subject, method);
- workers[1].reset(&index, &subject, method);
- sideThread.setTask(workers[0]);
- sideThread.start();
- workers[1].run();
- sideThread.join();
- } else {
- ConcurrentIndex index = ConcurrentIndex(begin, end);
- ParallelForWorker<Subject, Method, RangeType> * workers = new ParallelForWorker<Subject, Method,
- RangeType> [thread_num];
- Thread * threads = new Thread[thread_num - 1];
- for (size_t i = 0; i < thread_num - 1; i++) {
- workers[i].reset(&index, &subject, method);
- threads[i].setTask(workers[i]);
- threads[i].start();
- }
- workers[thread_num - 1].reset(&index, &subject, method);
- workers[thread_num - 1].run();
- for (size_t i = 0; i < thread_num - 1; i++) {
- threads[i].join();
- }
- delete[] threads;
- delete[] workers;
- }
-}
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc
index b1d3a00..9f8a9ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc
@@ -17,9 +17,9 @@
*/
#include <time.h>
-#include "commons.h"
-#include "StringUtil.h"
-#include "Timer.h"
+#include "lib/commons.h"
+#include "util/StringUtil.h"
+#include "util/Timer.h"
namespace NativeTask {
@@ -51,7 +51,6 @@ Timer::Timer() {
}
Timer::~Timer() {
-
}
uint64_t Timer::last() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
index a16c654..2436681 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "StringUtil.h"
-#include "WritableUtils.h"
+#include "lib/commons.h"
+#include "util/StringUtil.h"
+#include "util/WritableUtils.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h
index 3e93cbe..24a2b1b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h
@@ -21,7 +21,7 @@
#include <stdint.h>
#include <string>
-#include "Streams.h"
+#include "lib/Streams.h"
#include "NativeTask.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
index bcc3b3c..c09b7b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
@@ -16,13 +16,13 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "BufferStream.h"
-#include "Buffers.h"
+#include "lib/commons.h"
+#include "lib/BufferStream.h"
+#include "lib/Buffers.h"
#include "test_commons.h"
#include "NativeTask.h"
-using namespace NativeTask;
+namespace NativeTask {
TEST(Command, equals) {
Command cmd1(100, "hello command");
@@ -36,3 +36,4 @@ TEST(Command, equals) {
ASSERT_EQ(helloCommand, cmd1.description());
}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
index 1b99286..387e262 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
@@ -18,11 +18,11 @@
#include "lz4.h"
#include "config.h"
-#include "commons.h"
-#include "Path.h"
-#include "BufferStream.h"
-#include "FileSystem.h"
-#include "Compressions.h"
+#include "lib/commons.h"
+#include "lib/Path.h"
+#include "lib/BufferStream.h"
+#include "lib/FileSystem.h"
+#include "lib/Compressions.h"
#include "test_commons.h"
#if defined HADOOP_SNAPPY_LIBRARY
@@ -144,7 +144,7 @@ TEST(Perf, CompressionUtil) {
}
class CompressResult {
-public:
+ public:
uint64_t uncompressedSize;
uint64_t compressedSize;
uint64_t compressTime;
@@ -193,10 +193,8 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl
result.compressTime += endTime - startTime;
startTime = t.now();
for (int i = 0; i < times; i++) {
-// memset(dest, 0, currentblocksize+8);
int osize = LZ4_uncompress(outputBuffer, dest, currentblocksize);
ASSERT_EQ(currentblocksize, osize);
-// printf("%016llx blocksize: %lu\n", bswap64(*(uint64_t*)(dest+currentblocksize)), currentblocksize);
}
endTime = t.now();
result.uncompressTime += endTime - startTime;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc
index 36315ce..a5c0b9a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "BufferStream.h"
-#include "Buffers.h"
+#include "lib/commons.h"
+#include "lib/BufferStream.h"
+#include "lib/Buffers.h"
#include "test_commons.h"
float absoute(float v) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
index d4550bc..028c43b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "NativeObjectFactory.h"
-#include "BufferStream.h"
-#include "Buffers.h"
+#include "lib/commons.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/BufferStream.h"
+#include "lib/Buffers.h"
#include "test_commons.h"
TEST(Counter, Counter) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
index 8b5a76c..667858b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "FileSystem.h"
+#include "lib/FileSystem.h"
#include "test_commons.h"
TEST(FileSystem, RawFileSystem) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 955ef29..e1e32d4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -17,11 +17,11 @@
*/
#include <algorithm>
-#include "commons.h"
+#include "lib/commons.h"
#include "config.h"
-#include "BufferStream.h"
-#include "FileSystem.h"
-#include "IFile.h"
+#include "lib/BufferStream.h"
+#include "lib/FileSystem.h"
+#include "lib/IFile.h"
#include "test_commons.h"
SingleSpillInfo * writeIFile(int partition, vector<pair<string, string> > & kvs,
@@ -166,8 +166,8 @@ TEST(Perf, IFile) {
delete[] buff;
}
-// The Glibc has a bug in the file tell api, it will overwrite the file data
-// unexpected.
+// The Glibc has a bug in the file tell api, it will overwrite the file data
+// unexpected.
// Please check https://rhn.redhat.com/errata/RHBA-2013-0279.html
// This case is to check whether the bug exists.
// If it exists, it means you need to upgrade the glibc.
@@ -189,12 +189,11 @@ TEST(IFile, TestGlibCBug) {
uint32_t length = 0;
reader->nextPartition();
uint32_t index = 0;
- while(NULL != (key = reader->nextKey(length))) {
+ while (NULL != (key = reader->nextKey(length))) {
int32_t realKey = (int32_t)bswap(*(uint32_t *)(key));
ASSERT_LT(index, 5);
ASSERT_EQ(expect[index], realKey);
index++;
}
delete reader;
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
index c62232a..8af4c8a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
@@ -23,10 +23,10 @@
#endif
#include <stdexcept>
-#include "commons.h"
-#include "Buffers.h"
-#include "FileSystem.h"
-#include "NativeObjectFactory.h"
+#include "lib/commons.h"
+#include "lib/Buffers.h"
+#include "lib/FileSystem.h"
+#include "lib/NativeObjectFactory.h"
#include "test_commons.h"
extern "C" {
@@ -52,11 +52,11 @@ void handler(int sig) {
}
}
-using namespace NativeTask;
-
typedef char * CString;
-int main(int argc, char ** argv) {
+namespace NativeTask {
+
+int DoMain(int argc, char** argv) {
signal(SIGSEGV, handler);
CString * newArgv = new CString[argc + 1];
memcpy(newArgv, argv, argc * sizeof(CString));
@@ -107,3 +107,10 @@ int main(int argc, char ** argv) {
return 1;
}
}
+
+} // namespace NativeTask
+
+
+int main(int argc, char ** argv) {
+ return NativeTask::DoMain(argc, argv);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc
index 5a549c5..b2051c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc
@@ -166,7 +166,7 @@ inline char * memchrbrf2(char * p, char ch, size_t len) {
// not safe in MACOSX, segment fault, should be safe on Linux with out mmap
inline int memchr_sse(const char *s, int c, int len) {
- //len : edx; c: esi; s:rdi
+ // len : edx; c: esi; s:rdi
int index = 0;
#ifdef __X64
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc
index 2403db9..1c391a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "Streams.h"
-#include "Buffers.h"
-#include "DualPivotQuickSort.h"
+#include "lib/commons.h"
+#include "lib/Streams.h"
+#include "lib/Buffers.h"
+#include "util/DualPivotQuickSort.h"
#include "test_commons.h"
string gBuffer;
@@ -82,7 +82,7 @@ static int compare_offset(const void * plh, const void * prh) {
* dualpivot sort compare function
*/
class CompareOffset {
-public:
+ public:
int64_t operator()(uint32_t lhs, uint32_t rhs) {
KVBuffer * lhb = (KVBuffer*)get_position(lhs);
@@ -101,7 +101,7 @@ public:
* quicksort compare function
*/
class OffsetLessThan {
-public:
+ public:
bool operator()(uint32_t lhs, uint32_t rhs) {
KVBuffer * lhb = (KVBuffer*)get_position(lhs);
KVBuffer * rhb = (KVBuffer*)get_position(rhs);
@@ -132,7 +132,7 @@ static int compare_offset2(const void * plh, const void * prh) {
* dualpivot sort compare function
*/
class CompareOffset2 {
-public:
+ public:
int64_t operator()(uint32_t lhs, uint32_t rhs) {
KVBuffer * lhb = (KVBuffer*)get_position(lhs);
@@ -151,7 +151,7 @@ public:
* quicksort compare function
*/
class OffsetLessThan2 {
-public:
+ public:
bool operator()(uint32_t lhs, uint32_t rhs) {
KVBuffer * lhb = (KVBuffer*)get_position(lhs);
@@ -163,23 +163,6 @@ public:
}
};
-/*
-void makeInput(string & dest, vector<uint32_t> & offsets, uint64_t length) {
- TeraGen tera = TeraGen(length / 100, 1, 0);
- dest.reserve(length + 1024);
- string k, v;
- while (tera.next(k, v)) {
- offsets.push_back(dest.length());
- uint32_t tempLen = k.length();
- dest.append((const char *)&tempLen, 4);
- dest.append(k.data(), k.length());
- tempLen = v.length();
- dest.append((const char *)&tempLen, 4);
- dest.append(v.data(), v.length());
- }
-}
-*/
-
void makeInputWord(string & dest, vector<uint32_t> & offsets, uint64_t length) {
Random r;
dest.reserve(length + 1024);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc
index f20d67f..1b26737 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
index c03f02e..02acdda 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
index 9572d87..f9e980e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "lib/MemoryBlock.h"
#include "test_commons.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
index 6dd97ba..1a914b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "lib/MemoryBlock.h"
#include "test_commons.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
index a2dd909..d1b5c37 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
index 88a3258..d25cd60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
@@ -30,7 +30,7 @@ class MockIterator : public KVIterator {
std::map<int, int> expectkeyCountMap;
char buffer[8];
-public:
+ public:
MockIterator()
: index(0) {
kvs.push_back(std::pair<int, int>(10, 100));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
index 32b5891..e47e169 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
index c11d5ec..8d784fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
@@ -16,13 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "test_commons.h"
-#include "MapOutputSpec.h"
+#include "lib/MapOutputSpec.h"
#include "lib/MemoryBlock.h"
-using namespace NativeTask;
-
namespace NativeTask {
TEST(MemoryBlockIterator, test) {
@@ -46,7 +44,7 @@ TEST(MemoryBlockIterator, test) {
}
class MemoryBlockFactory {
-public:
+ public:
static MemoryBlock * create(std::vector<int> & keys) {
const uint32_t BUFFER_LENGTH = 1000;
char * bytes = new char[BUFFER_LENGTH];
@@ -65,7 +63,6 @@ public:
}
return block1;
}
-
};
TEST(MemoryBlockIterator, compare) {
@@ -109,5 +106,5 @@ TEST(MemoryBlockIterator, compare) {
delete block2;
delete block1;
}
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
index 1d48c79..6af73c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
@@ -16,12 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "test_commons.h"
-#include "MapOutputSpec.h"
+#include "lib/MapOutputSpec.h"
#include "lib/MemoryBlock.h"
-using namespace NativeTask;
namespace NativeTaskTest {
TEST(MemoryBlock, test) {
@@ -107,5 +106,4 @@ TEST(MemoryBlock, sort) {
delete [] bytes;
}
-}
-
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc
index 3c67332..e33421d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc
@@ -16,14 +16,12 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "test_commons.h"
-#include "PartitionBucket.h"
-#include "PartitionBucketIterator.h"
-#include "MemoryBlock.h"
-#include "IFile.h"
-
-using namespace NativeTask;
+#include "lib/PartitionBucket.h"
+#include "lib/PartitionBucketIterator.h"
+#include "lib/MemoryBlock.h"
+#include "lib/IFile.h"
namespace NativeTask {
@@ -47,4 +45,4 @@ TEST(MemoryPool, general) {
delete pool;
}
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
index a975243..79e1b5e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
@@ -16,24 +16,22 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "test_commons.h"
-#include "PartitionBucket.h"
-#include "PartitionBucketIterator.h"
-#include "MemoryBlock.h"
-#include "IFile.h"
-
-using namespace NativeTask;
+#include "lib/PartitionBucket.h"
+#include "lib/PartitionBucketIterator.h"
+#include "lib/MemoryBlock.h"
+#include "lib/IFile.h"
namespace NativeTask {
class MockIFileWriter : public IFileWriter {
-private:
+ private:
char * _buff;
uint32_t _position;
uint32_t _capacity;
-public:
+ public:
MockIFileWriter(char * buff, uint32_t capacity)
: IFileWriter(NULL, CHECKSUM_NONE, TextType, TextType, "", NULL), _buff(buff), _position(0),
_capacity(capacity) {
@@ -55,8 +53,8 @@ public:
TEST(PartitionBucket, general) {
MemoryPool * pool = new MemoryPool();
- const uint32_t POOL_SIZE = 1024 * 1024; //1MB
- const uint32_t BLOCK_SIZE = 1024; //1KB
+ const uint32_t POOL_SIZE = 1024 * 1024; // 1MB
+ const uint32_t BLOCK_SIZE = 1024; // 1KB
const uint32_t PARTITION_ID = 3;
pool->init(POOL_SIZE);
ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL);
@@ -74,8 +72,8 @@ TEST(PartitionBucket, general) {
TEST(PartitionBucket, multipleMemoryBlock) {
MemoryPool * pool = new MemoryPool();
- const uint32_t POOL_SIZE = 1024 * 1024; //1MB
- const uint32_t BLOCK_SIZE = 1024; //1KB
+ const uint32_t POOL_SIZE = 1024 * 1024; // 1MB
+ const uint32_t BLOCK_SIZE = 1024; // 1KB
const uint32_t PARTITION_ID = 3;
pool->init(POOL_SIZE);
ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL);
@@ -113,8 +111,8 @@ TEST(PartitionBucket, multipleMemoryBlock) {
TEST(PartitionBucket, sort) {
MemoryPool * pool = new MemoryPool();
- const uint32_t POOL_SIZE = 1024 * 1024; //1MB
- const uint32_t BLOCK_SIZE = 1024; //1KB
+ const uint32_t POOL_SIZE = 1024 * 1024; // 1MB
+ const uint32_t BLOCK_SIZE = 1024; // 1KB
const uint32_t PARTITION_ID = 3;
pool->init(POOL_SIZE);
ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL);
@@ -165,8 +163,8 @@ TEST(PartitionBucket, sort) {
TEST(PartitionBucket, spill) {
MemoryPool * pool = new MemoryPool();
- const uint32_t POOL_SIZE = 1024 * 1024; //1MB
- const uint32_t BLOCK_SIZE = 1024; //1KB
+ const uint32_t POOL_SIZE = 1024 * 1024; // 1MB
+ const uint32_t BLOCK_SIZE = 1024; // 1KB
const uint32_t PARTITION_ID = 3;
pool->init(POOL_SIZE);
ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL);
@@ -201,7 +199,7 @@ TEST(PartitionBucket, spill) {
MockIFileWriter writer(buff, BUFF_SIZE);
bucket->spill(&writer);
- //check the result
+ // check the result
KVBuffer * first = (KVBuffer *)writer.buff();
ASSERT_EQ(4, first->keyLength);
ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, first->valueLength);
@@ -221,4 +219,4 @@ TEST(PartitionBucket, spill) {
delete bucket;
delete pool;
}
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
index cbb636f..1a4b290 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "config.h"
-#include "BufferStream.h"
-#include "Buffers.h"
+#include "lib/BufferStream.h"
+#include "lib/Buffers.h"
#include "test_commons.h"
TEST(Buffers, AppendRead) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
index efbb8ea..f32989c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lib/Combiner.h"
#include "test_commons.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc
deleted file mode 100644
index e4f7c64..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "commons.h"
-#include "lib/Combiner.h"
-#include "test_commons.h"
-#include <iostream>
-
-namespace NativeTask {
-
-TEST(TrackingCollector, read) {
- const std::string GROUP("group");
- const std::string KEY("key");
- Counter * counter = new Counter(GROUP, KEY);
- Collector * collector = new Collector();
- TrackingCollector tracking(collector, counter);
- tracking.collect(NULL, 0, NULL, 0);
- ASSERT_EQ(1, counter->get());
- delete counter;
- delete collector;
-}
-} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc
index 4ec1f48..f7332be 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc
@@ -17,9 +17,9 @@
*/
#include <stdarg.h>
-#include "commons.h"
+#include "lib/commons.h"
#include "util/Random.h"
-#include "FileSystem.h"
+#include "lib/FileSystem.h"
#include "test_commons.h"
Config TestConfig = Config();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h
index 48884ce..90d48ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h
@@ -24,8 +24,8 @@
#include "util/Random.h"
#include "util/StringUtil.h"
#include "util/Timer.h"
-#include "Buffers.h"
-#include "BufferStream.h"
+#include "lib/Buffers.h"
+#include "lib/BufferStream.h"
using std::pair;
using std::vector;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
deleted file mode 100644
index 501e005..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "util/Hash.h"
-#include "test_commons.h"
-
-static uint64_t test_length(int64_t len, size_t size, size_t loopTime) {
- vector<string> data;
- TestConfig.setInt(GenerateLen, len);
- Generate(data, size, "bytes");
- Timer t;
- uint64_t ret = 0;
- for (size_t m = 0; m < loopTime; m++) {
- for (size_t i = 0; i < data.size(); i++) {
- ret += Hash::BytesHash(data[i].c_str(), data[i].length());
- }
- }
- LOG("%s", t.getInterval(StringUtil::Format("Bytes%3lld", len).c_str()).c_str());
- t.reset();
- for (size_t m = 0; m < loopTime; m++) {
- for (size_t i = 0; i < data.size(); i++) {
- ret += Hash::CityHash(data[i].c_str(), data[i].length());
- }
- }
- LOG("%s", t.getInterval(StringUtil::Format(" City%3lld", len).c_str()).c_str());
- return ret;
-}
-
-TEST(Perf, Hash) {
- uint64_t ret = 0;
- ret += test_length(1, 100, 4000);
- ret += test_length(17, 100, 4000);
- ret += test_length(64, 100, 4000);
- ret += test_length(128, 100, 4000);
- ret += test_length(513, 100, 4000);
- fprintf(stderr, "%llu\n", (long long unsigned int)ret);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc
deleted file mode 100644
index 7641f9c..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "util/SyncUtils.h"
-#include "test_commons.h"
-
-class TestThread : public Thread {
- virtual void run() {
- for (uint32_t i = 0; i < 5; i++) {
- usleep(100);
- LOG("sleep %d", i * 100);
- }
- }
-};
-
-TEST(SyncUtil, Thread) {
- TestThread a, b, c;
- a.start();
- b.start();
- c.start();
- a.join();
- b.join();
- c.join();
-}
-
-class TestBind {
-public:
-
- int get() {
- return 100;
- }
-
- void foo() {
- for (uint32_t i = 0; i < 2; i++) {
- usleep(100);
- LOG("usleep %d", i * 100);
- }
- }
- void bar(const char * msg) {
- for (uint32_t i = 0; i < 2; i++) {
- usleep(100);
- LOG("usleep %d %s", i * 100, msg);
- }
- }
-
-};
-
-TEST(SyncUtil, ThreadBind) {
- TestBind a = TestBind();
- Runnable * bind1 = BindNew(a, &TestBind::get);
- Thread t = Thread(bind1);
- Runnable * bind2 = BindNew(a, &TestBind::bar, "testmsg");
- Thread t2 = Thread(bind2);
- t.start();
- t2.start();
- t.join();
- t2.join();
-
- delete bind1;
- delete bind2;
-}
-
-//class TestParallelFor {
-//protected:
-// SpinLock lock;
-// uint64_t aggr;
-//public:
-// TestParallelFor() : aggr(0) {
-// }
-// void add(uint64_t i) {
-// lock.lock();
-// aggr += i;
-// lock.unlock();
-// }
-// void test(uint64_t n, size_t threadnum) {
-// aggr = 0;
-// ParallelFor(*this, &TestParallelFor::add, 0ULL, n, threadnum);
-// ASSERT_EQ(n*(n-1)/2, aggr);
-// }
-//};
-//
-//TEST(SyncUtil, ParallelFor) {
-// TestParallelFor tpf;
-// tpf.test(100000, 2);
-// tpf.test(100000, 3);
-// tpf.test(100000, 4);
-//}
-
-TEST(Perf, ThreadOverhead) {
- int64_t threadnum = TestConfig.getInt("thread.num", 1000);
- Thread * t = new Thread[threadnum];
- Timer timer;
- for (uint32_t i = 0; i < threadnum; i++) {
- t[i].start();
- }
- for (uint32_t i = 0; i < threadnum; i++) {
- t[i].join();
- }
- LOG("%lld thread %s", (long long int )threadnum, timer.getInterval("start&join").c_str());
- delete[] t;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index 61b4356..3aa8a07 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -86,8 +86,10 @@ public class CombinerTest {
Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
}
- this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR + "/nativewordcount";
- this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR + "/normalwordcount";
+ this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR +
+ "/nativewordcount";
+ this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR +
+ "/normalwordcount";
}
@AfterClass
@@ -97,7 +99,8 @@ public class CombinerTest {
fs.close();
}
- protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
+ protected static Job getJob(String jobname, Configuration inputConf,
+ String inputpath, String outputpath)
throws Exception {
final Configuration conf = new Configuration(inputConf);
conf.set("fileoutputpath", outputpath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index d054793..a4eecbe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -84,8 +84,10 @@ public class LargeKVCombinerTest {
1000000), IntWritable.class.getName(),
Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
- final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
- final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
+ final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf,
+ inputPath, hadoopOutputPath);
+ final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf,
+ inputPath, nativeOutputPath);
assertTrue(nativejob.waitForCompletion(true));
@@ -93,8 +95,9 @@ public class LargeKVCombinerTest {
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
- final String reason = "LargeKVCombinerTest failed with, min size: " + min
- + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
+ final String reason = "LargeKVCombinerTest failed with, min size: " + min +
+ ", max size: " + max + ", normal out: " + hadoopOutputPath +
+ ", native Out: " + nativeOutputPath;
assertEquals(reason, true, compareRet);
ResultVerifier.verifyCounters(normaljob, nativejob, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
index 1c76bc6..c3fc02f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
@@ -55,23 +55,28 @@ public class OldAPICombinerTest {
final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
final String nativeoutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH;
- final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
+ final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI",
+ inputpath, nativeoutput);
RunningJob nativeRunning = JobClient.runJob(nativeJob);
- Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+ Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(
+ Task.Counter.REDUCE_INPUT_RECORDS);
final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
final String normaloutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH;
- final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
+ final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI",
+ inputpath, normaloutput);
RunningJob normalRunning = JobClient.runJob(normalJob);
- Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+ Counter normalReduceGroups = normalRunning.getCounters().findCounter(
+ Task.Counter.REDUCE_INPUT_RECORDS);
final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+ assertEquals("The input reduce record count must be same",
+ nativeReduceGroups.getValue(), normalReduceGroups.getValue());
}
@Before
@@ -84,7 +89,8 @@ public class OldAPICombinerTest {
this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount";
if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000), Text.class.getName(),
+ new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000),
+ Text.class.getName(),
Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
}
}
@@ -96,7 +102,8 @@ public class OldAPICombinerTest {
fs.close();
}
- private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
+ private static JobConf getOldAPIJobconf(Configuration configuration, String name,
+ String input, String output)
throws Exception {
final JobConf jobConf = new JobConf(configuration);
final FileSystem fs = FileSystem.get(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
index 490b82b..7b33672 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
@@ -44,7 +44,8 @@ public class WordCount {
private final Text word = new Text();
@Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+ public void map(Object key, Text value, Context context)
+ throws IOException, InterruptedException {
final StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
index a11ea91..00547b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
@@ -37,7 +37,9 @@ public class WordCountWithOldAPI {
private final Text word = new Text();
@Override
- public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ public void map(Object key, Text value,
+ OutputCollector<Text, IntWritable> output,
+ Reporter reporter)
throws IOException {
final StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
@@ -52,8 +54,9 @@ public class WordCountWithOldAPI {
private final IntWritable result = new IntWritable();
@Override
- public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
+ public void reduce(Text key, Iterator<IntWritable> values,
+ OutputCollector<Text, IntWritable> output,
+ Reporter reporter) throws IOException {
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
index 1c91ff0..8198b47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
@@ -35,12 +35,14 @@ public class CompressMapper {
public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
@Override
- protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
+ protected void map(Text key, Text value, Context context)
+ throws IOException, InterruptedException {
context.write(key, value);
}
}
- public static Job getCompressJob(String jobname, Configuration conf, String inputpath, String outputpath)
+ public static Job getCompressJob(String jobname, Configuration conf,
+ String inputpath, String outputpath)
throws Exception {
Job job = new Job(conf, jobname + "-CompressMapperJob");
job.setJarByClass(CompressMapper.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
index 7b337b7..81d632d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java
@@ -41,13 +41,13 @@ public class TestCombineHandler extends TestCase {
@Override
public void setUp() throws IOException {
-
this.nativeHandler = Mockito.mock(INativeHandler.class);
this.pusher = Mockito.mock(BufferPusher.class);
this.puller = Mockito.mock(BufferPuller.class);
this.combinerRunner = Mockito.mock(CombinerRunner.class);
- Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100));
+ Mockito.when(nativeHandler.getInputBuffer()).thenReturn(
+ new InputBuffer(BufferType.HEAP_BUFFER, 100));
}
public void testCommandDispatcherSetting() throws IOException {
@@ -62,7 +62,8 @@ public class TestCombineHandler extends TestCase {
handler.close();
handler.close();
- Mockito.verify(combinerRunner, Mockito.times(1)).combine(Matchers.eq(puller), Matchers.eq(pusher));
+ Mockito.verify(combinerRunner, Mockito.times(1))
+ .combine(Matchers.eq(puller), Matchers.eq(pusher));
Mockito.verify(pusher, Mockito.times(1)).close();
Mockito.verify(puller, Mockito.times(1)).close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
index 6b9edc8..6c15360 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
@@ -65,7 +65,8 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
null,
null);
- Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100));
+ Mockito.when(nativeHandler.getInputBuffer()).thenReturn(
+ new InputBuffer(BufferType.HEAP_BUFFER, 100));
}
@Override
@@ -91,7 +92,8 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
public void testGetCombiner() throws IOException {
this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
Mockito.when(combiner.getId()).thenReturn(100L);
- final ReadWriteBuffer result = handler.onCall(NativeCollectorOnlyHandler.GET_COMBINE_HANDLER, null);
+ final ReadWriteBuffer result = handler.onCall(
+ NativeCollectorOnlyHandler.GET_COMBINE_HANDLER, null);
Assert.assertEquals(100L, result.readLong());
}
@@ -109,13 +111,16 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
final String expectedOutputIndexPath = LOCAL_DIR + "/output/file.out.index";
final String expectedSpillPath = LOCAL_DIR + "/output/spill0.out";
- final String outputPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString();
+ final String outputPath = handler.onCall(
+ NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString();
Assert.assertEquals(expectedOutputPath, outputPath);
- final String outputIndexPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_INDEX_PATH, null).readString();
+ final String outputIndexPath = handler.onCall(
+ NativeCollectorOnlyHandler.GET_OUTPUT_INDEX_PATH, null).readString();
Assert.assertEquals(expectedOutputIndexPath, outputIndexPath);
- final String spillPath = handler.onCall(NativeCollectorOnlyHandler.GET_SPILL_PATH, null).readString();
+ final String spillPath = handler.onCall(
+ NativeCollectorOnlyHandler.GET_SPILL_PATH, null).readString();
Assert.assertEquals(expectedSpillPath, spillPath);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
index ce23780..2a72d05 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
@@ -32,7 +32,8 @@ public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, I
DataOutputStream dos = new DataOutputStream(os);
@Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
+ public void reduce(KTYPE key, Iterable<VTYPE> values, Context context)
+ throws IOException, InterruptedException {
int hashSum = 0;
for (final VTYPE val : values) {
if (val instanceof Writable) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 9e7bb5e..0a4c955 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -45,13 +45,15 @@ public class KVJob {
public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
@Override
- public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
+ public void map(KTYPE key, VTYPE value, Context context)
+ throws IOException, InterruptedException {
context.write(key, value);
}
}
public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
- public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
+ public void reduce(KTYPE key, VTYPE value, Context context)
+ throws IOException, InterruptedException {
context.write(key, value);
}
}
@@ -59,7 +61,8 @@ public class KVJob {
public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
@Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
+ public void reduce(KTYPE key, Iterable<VTYPE> values, Context context)
+ throws IOException, InterruptedException {
long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
final CRC32 crc32 = new CRC32();
for (final VTYPE val : values) {
@@ -68,12 +71,14 @@ public class KVJob {
resultlong += crc32.getValue();
}
final VTYPE V = null;
- context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
+ context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong),
+ V.getClass().getName()));
}
}
- public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath,
- String outputpath) throws Exception {
+ public KVJob(String jobname, Configuration conf,
+ Class<?> keyclass, Class<?> valueclass,
+ String inputpath, String outputpath) throws Exception {
job = new Job(conf, jobname);
job.setJarByClass(KVJob.class);
job.setMapperClass(KVJob.ValueMapper.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index 4c77f4c..49331ce 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -89,7 +89,8 @@ public class TestInputFile {
map.put(IntWritable.class.getName(), new KVSizeScope(4, 4));
}
- public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception {
+ public TestInputFile(int filesize, String keytype, String valuetype,
+ Configuration conf) throws Exception {
this.filesize = filesize;
this.databuf = new byte[DATABUFSIZE];
this.keyClsName = keytype;
[35/50] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of
unused code. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index 2258726..f81d94f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -74,13 +74,14 @@ public class NonSortTest {
public void startUp() throws Exception {
Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
- final ScenarioConfiguration configuration = new ScenarioConfiguration();
- configuration.addNonSortTestConf();
- final FileSystem fs = FileSystem.get(configuration);
+ final ScenarioConfiguration conf = new ScenarioConfiguration();
+ conf.addNonSortTestConf();
+ final FileSystem fs = FileSystem.get(conf);
final Path path = new Path(TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR);
if (!fs.exists(path)) {
- new TestInputFile(configuration.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE, 10000000), Text.class.getName(),
- Text.class.getName(), configuration).createSequenceTestFile(path.toString());
+ int filesize = conf.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE, 10000000);
+ new TestInputFile(filesize, Text.class.getName(),
+ Text.class.getName(), conf).createSequenceTestFile(path.toString());
}
fs.close();
}
@@ -93,7 +94,8 @@ public class NonSortTest {
}
- private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
+ private Job getJob(Configuration conf, String jobName,
+ String inputpath, String outputpath) throws IOException {
final FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputpath))) {
fs.delete(new Path(outputpath), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
index 4ca2449..4092e5f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
@@ -36,7 +36,8 @@ public class NonSortTestMR {
private final Text word = new Text();
@Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+ public void map(Object key, Text value, Context context)
+ throws IOException, InterruptedException {
final String line = value.toString();
final StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
index 004e8b8..fd5b100 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
@@ -79,13 +79,15 @@ public class TestKVSerializer extends TestCase {
final DataOutputStream dataOut = Mockito.mock(DataOutputStream.class);
Mockito.when(dataOut.hasUnFlushedData()).thenReturn(true);
- Mockito.when(dataOut.shortOfSpace(key.length + value.length + Constants.SIZEOF_KV_LENGTH)).thenReturn(true);
+ Mockito.when(dataOut.shortOfSpace(key.length + value.length +
+ Constants.SIZEOF_KV_LENGTH)).thenReturn(true);
final int written = serializer.serializeKV(dataOut, key, value);
// flush once, write 4 int, and 2 byte array
Mockito.verify(dataOut, Mockito.times(1)).flush();
Mockito.verify(dataOut, Mockito.times(4)).writeInt(Matchers.anyInt());
- Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class), Matchers.anyInt(), Matchers.anyInt());
+ Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class),
+ Matchers.anyInt(), Matchers.anyInt());
Assert.assertEquals(written, key.length + value.length + Constants.SIZEOF_KV_LENGTH);
}
@@ -101,7 +103,8 @@ public class TestKVSerializer extends TestCase {
// flush 0, write 4 int, and 2 byte array
Mockito.verify(dataOut, Mockito.times(0)).flush();
Mockito.verify(dataOut, Mockito.times(4)).writeInt(Matchers.anyInt());
- Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class), Matchers.anyInt(), Matchers.anyInt());
+ Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class),
+ Matchers.anyInt(), Matchers.anyInt());
Assert.assertEquals(written, key.length + value.length + Constants.SIZEOF_KV_LENGTH);
}
@@ -112,14 +115,16 @@ public class TestKVSerializer extends TestCase {
Mockito.when(dataOut.hasUnFlushedData()).thenReturn(true);
Mockito.when(
dataOut
- .shortOfSpace(key.length + value.length + Constants.SIZEOF_KV_LENGTH + Constants.SIZEOF_PARTITION_LENGTH))
+ .shortOfSpace(key.length + value.length +
+ Constants.SIZEOF_KV_LENGTH + Constants.SIZEOF_PARTITION_LENGTH))
.thenReturn(true);
final int written = serializer.serializePartitionKV(dataOut, 100, key, value);
// flush once, write 4 int, and 2 byte array
Mockito.verify(dataOut, Mockito.times(1)).flush();
Mockito.verify(dataOut, Mockito.times(5)).writeInt(Matchers.anyInt());
- Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class), Matchers.anyInt(), Matchers.anyInt());
+ Mockito.verify(dataOut, Mockito.times(2)).write(Matchers.any(byte[].class),
+ Matchers.anyInt(), Matchers.anyInt());
Assert.assertEquals(written, key.length + value.length + Constants.SIZEOF_KV_LENGTH
+ Constants.SIZEOF_PARTITION_LENGTH);
@@ -137,6 +142,7 @@ public class TestKVSerializer extends TestCase {
Assert.assertTrue(serializer.deserializeKV(in, key, value) > 0);
Mockito.verify(in, Mockito.times(4)).readInt();
- Mockito.verify(in, Mockito.times(2)).readFully(Matchers.any(byte[].class), Matchers.anyInt(), Matchers.anyInt());
+ Mockito.verify(in, Mockito.times(2)).readFully(Matchers.any(byte[].class),
+ Matchers.anyInt(), Matchers.anyInt());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
index 4b67454..cf708a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestNativeSerialization.java
@@ -93,7 +93,8 @@ public class TestNativeSerialization extends TestCase {
public static class NonWritableValue {
}
- public static class ComparableKeySerializer implements INativeComparable, INativeSerializer<WritableKey> {
+ public static class ComparableKeySerializer
+ implements INativeComparable, INativeSerializer<WritableKey> {
@Override
public int getLength(WritableKey w) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
index 18ca629..7212236 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
@@ -23,7 +23,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
-public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
+public class EnforceNativeOutputCollectorDelegator<K, V>
+ extends NativeMapOutputCollectorDelegator<K, V> {
private static final Log LOG = LogFactory.getLog(EnforceNativeOutputCollectorDelegator.class);
private boolean nativetaskloaded = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
index 8a7916b..5749691 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
@@ -35,11 +35,8 @@ public class ResultVerifier {
/**
* verify the result
*
- * @param sample
- * :nativetask output
- * @param source
- * :yuanwenjian
- * @throws Exception
+ * @param sample the path to correct results
+ * @param source the path to the results from the native implementation
*/
public static boolean verify(String sample, String source) throws Exception {
FSDataInputStream sourcein = null;
@@ -76,8 +73,9 @@ public class ResultVerifier {
}
}
if (samplepath == null) {
- throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName()
- + " , as sourcepaths has such file");
+ throw new Exception("cound not find file " +
+ samplepaths[0].getParent() + "/" + sourcepath.getName()
+ + " , as sourcepaths has such file");
}
// compare
@@ -141,7 +139,8 @@ public class ResultVerifier {
return true;
}
- public static void verifyCounters(Job normalJob, Job nativeJob, boolean hasCombiner) throws IOException {
+ public static void verifyCounters(Job normalJob, Job nativeJob, boolean hasCombiner)
+ throws IOException {
Counters normalCounters = normalJob.getCounters();
Counters nativeCounters = nativeJob.getCounters();
assertEquals("Counter MAP_OUTPUT_RECORDS should be equal",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestInput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestInput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestInput.java
index 6713a4a..23fb1ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestInput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestInput.java
@@ -29,10 +29,11 @@ public class TestInput {
public V value;
}
- public static char[] CHAR_SET = new char[] { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
- 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
- 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5',
- '6', '7', '8', '9', '*', '/' };
+ public static char[] CHAR_SET = new char[] {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',
+ 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
+ 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',
+ 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*', '/' };
public static KV[] getMapInputs(int size) {
[28/50] [abbrv] git commit: MAPREDUCE-6058. native-task: KVTest and
LargeKVTest should check mr job is sucessful (Contributed by Binglin Chang)
Posted by ar...@apache.org.
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Contributed by Binglin Chang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17cd0faa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17cd0faa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17cd0faa
Branch: refs/heads/HDFS-6581
Commit: 17cd0faaceb8f9ce00b8c2c200e810f78f36c671
Parents: bfd1d75
Author: Binglin Chang <bc...@apache.org>
Authored: Tue Sep 2 11:00:48 2014 +0800
Committer: Binglin Chang <bc...@apache.org>
Committed: Tue Sep 2 11:00:48 2014 +0800
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 28 ++------------------
.../nativetask/combinertest/CombinerTest.java | 10 +++++--
.../combinertest/LargeKVCombinerTest.java | 15 +++++++++--
.../combinertest/OldAPICombinerTest.java | 5 ++++
.../nativetask/compresstest/CompressTest.java | 18 ++++++++-----
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 5 ++--
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 15 +++++++++--
.../mapred/nativetask/kvtest/LargeKVTest.java | 15 +++++++++--
.../nativetask/nonsorttest/NonSortTest.java | 5 ++++
10 files changed, 74 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 6384757..baa88c1 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -17,3 +17,4 @@ MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd)
MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
MAPREDUCE-6054. native-task: Speed up tests (todd)
+MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index bb7d7bb..f62743e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -83,10 +83,8 @@
</property>
</properties>
<includes>
- <include>**/TestTaskContext.java</include>
- <include>**/buffer/Test*.java</include>
- <include>**/handlers/Test*.java</include>
- <include>**/serde/Test*.java</include>
+ <include>**/*Test.java</include>
+ <include>**/Test*.java</include>
</includes>
</configuration>
</plugin>
@@ -201,28 +199,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <properties>
- <property>
- <name>listener</name>
- <value>org.apache.hadoop.test.TimedOutTestsListener</value>
- </property>
- </properties>
- <includes>
- <include>**/TestTaskContext.java</include>
- <include>**/buffer/Test*.java</include>
- <include>**/handlers/Test*.java</include>
- <include>**/serde/Test*.java</include>
- <include>**/combinertest/*Test.java</include>
- <include>**/compresstest/*Test.java</include>
- <include>**/nonsorttest/*Test.java</include>
- <include>**/kvtest/*Test.java</include>
- </includes>
- </configuration>
- </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index 8a4aa6f..abbe28e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.nativetask.combinertest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -25,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
@@ -36,6 +38,8 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -58,11 +62,11 @@ public class CombinerTest {
final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
- nativejob.waitForCompletion(true);
+ assertTrue(nativejob.waitForCompletion(true));
Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
- normaljob.waitForCompletion(true);
+ assertTrue(normaljob.waitForCompletion(true));
Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
@@ -77,6 +81,8 @@ public class CombinerTest {
@Before
public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
final ScenarioConfiguration conf = new ScenarioConfiguration();
conf.addcombinerConf();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index ff444db..ccf0272 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.nativetask.combinertest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,17 +28,27 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
public class LargeKVCombinerTest {
private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class);
+ @Before
+ public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
+ }
+
@Test
public void testLargeValueCombiner(){
final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
@@ -74,10 +85,10 @@ public class LargeKVCombinerTest {
final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
- nativejob.waitForCompletion(true);
+ assertTrue(nativejob.waitForCompletion(true));
Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
- normaljob.waitForCompletion(true);
+ assertTrue(normaljob.waitForCompletion(true));
Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
index 5691e02..19493c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
@@ -32,11 +32,14 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.Task;
import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -70,6 +73,8 @@ public class OldAPICombinerTest {
@Before
public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
final ScenarioConfiguration conf = new ScenarioConfiguration();
conf.addcombinerConf();
this.fs = FileSystem.get(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index b98e2de..67321d6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -18,16 +18,20 @@
package org.apache.hadoop.mapred.nativetask.compresstest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -38,12 +42,12 @@ public class CompressTest {
final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
- job.waitForCompletion(true);
+ assertTrue(job.waitForCompletion(true));
final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
- hadoopjob.waitForCompletion(true);
+ assertTrue(hadoopjob.waitForCompletion(true));
final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
CompressMapper.outputFileDir + "hadoopsnappy");
@@ -55,12 +59,12 @@ public class CompressTest {
final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
final Job job = CompressMapper.getCompressJob("nativegzip", conf);
- job.waitForCompletion(true);
+ assertTrue(job.waitForCompletion(true));
final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
- hadoopjob.waitForCompletion(true);
+ assertTrue(hadoopjob.waitForCompletion(true));
final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
CompressMapper.outputFileDir + "hadoopgzip");
@@ -72,12 +76,12 @@ public class CompressTest {
final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
- nativeJob.waitForCompletion(true);
+ assertTrue(nativeJob.waitForCompletion(true));
final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
- hadoopJob.waitForCompletion(true);
+ assertTrue(hadoopJob.waitForCompletion(true));
final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
CompressMapper.outputFileDir + "hadooplz4");
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
@@ -85,6 +89,8 @@ public class CompressTest {
@Before
public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
final ScenarioConfiguration conf = new ScenarioConfiguration();
final FileSystem fs = FileSystem.get(conf);
final Path path = new Path(CompressMapper.inputFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 3215d0b..9e7bb5e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -96,8 +96,7 @@ public class KVJob {
FileOutputFormat.setOutputPath(job, new Path(outputpath));
}
- public void runJob() throws Exception {
-
- job.waitForCompletion(true);
+ public boolean runJob() throws Exception {
+ return job.waitForCompletion(true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 893a422..1657c04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.nativetask.kvtest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
@@ -28,9 +29,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -100,6 +105,12 @@ public class KVTest {
this.valueclass = valueclass;
}
+ @Before
+ public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
+ }
+
@Test
public void testKVCompability() {
try {
@@ -139,7 +150,7 @@ public class KVTest {
nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
try {
final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
+ assertTrue("job should complete successfully", keyJob.runJob());
} catch (final Exception e) {
return "native testcase run time error.";
}
@@ -161,7 +172,7 @@ public class KVTest {
hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
try {
final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
+ assertTrue("job should complete successfully", keyJob.runJob());
} catch (final Exception e) {
return "normal testcase run time error.";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
index 1e120ab..6b99907 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.nativetask.kvtest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -28,14 +29,24 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
public class LargeKVTest {
private static final Log LOG = LogFactory.getLog(LargeKVTest.class);
+ @Before
+ public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
+ }
+
@Test
public void testKeySize() {
runKVSizeTests(Text.class, IntWritable.class);
@@ -104,7 +115,7 @@ public class LargeKVTest {
fs.close();
try {
final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
+ assertTrue("job should complete successfully", keyJob.runJob());
} catch (final Exception e) {
return "normal testcase run time error.";
}
@@ -123,7 +134,7 @@ public class LargeKVTest {
fs.close();
try {
final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
+ assertTrue("job should complete successfully", keyJob.runJob());
} catch (final Exception e) {
return "normal testcase run time error.";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index b7b03e7..b1bb5b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
@@ -36,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -65,6 +68,8 @@ public class NonSortTest {
@Before
public void startUp() throws Exception {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
final ScenarioConfiguration configuration = new ScenarioConfiguration();
configuration.addNonSortTestConf();
final FileSystem fs = FileSystem.get(configuration);
[09/50] [abbrv] git commit: Merge trunk into branch
Posted by ar...@apache.org.
Merge trunk into branch
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613007 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5149a8a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5149a8a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5149a8a6
Branch: refs/heads/HDFS-6581
Commit: 5149a8a6f1d2fc673a7d011d7e0e1a5407ceac4c
Parents: b2cba48 2054453
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jul 24 06:23:41 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jul 24 06:23:41 2014 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 14 +
.../apache/hadoop/fs/ChecksumFileSystem.java | 4 +-
.../java/org/apache/hadoop/fs/FileContext.java | 29 ++
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 34 ++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 34 ++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 80 +++++
.../security/alias/JavaKeyStoreProvider.java | 5 +-
.../apache/hadoop/util/DirectBufferPool.java | 109 ++++++
.../org/apache/hadoop/fs/TestDFVariations.java | 21 +-
.../apache/hadoop/fs/shell/TestPathData.java | 12 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 30 ++
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 30 ++
.../hadoop/util/TestDirectBufferPool.java | 97 +++++
.../java/org/apache/hadoop/nfs/NfsExports.java | 7 +-
.../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 19 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 5 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 10 +
.../apache/hadoop/hdfs/BlockReaderLocal.java | 2 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 2 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
.../protocol/datatransfer/PacketReceiver.java | 2 +-
.../server/datanode/BlockPoolSliceScanner.java | 54 ++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 3 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 4 +
.../hdfs/server/namenode/FSEditLogOp.java | 5 +
.../hdfs/server/namenode/FSNamesystem.java | 62 +++-
.../server/namenode/XAttrPermissionFilter.java | 17 +-
.../hadoop/hdfs/util/DirectBufferPool.java | 107 ------
.../hdfs/web/resources/XAttrNameParam.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 69 ++++
.../hdfs/server/namenode/FSXAttrBaseTest.java | 351 +++++++++++++++++--
.../server/namenode/TestNamenodeRetryCache.java | 4 +-
.../namenode/ha/TestRetryCacheWithHA.java | 54 ++-
.../hadoop/hdfs/util/TestDirectBufferPool.java | 97 -----
.../hadoop/hdfs/web/resources/TestParam.java | 6 -
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4970 -> 4992 bytes
.../src/test/resources/editsStored.xml | 2 +
hadoop-yarn-project/CHANGES.txt | 12 +
.../dev-support/findbugs-exclude.xml | 6 +
.../BaseContainerTokenSecretManager.java | 2 +-
.../yarn/server/nodemanager/NodeManager.java | 6 +-
.../recovery/NMLeveldbStateStoreService.java | 117 ++++++-
.../recovery/NMNullStateStoreService.java | 30 +-
.../recovery/NMStateStoreService.java | 40 ++-
.../security/NMContainerTokenSecretManager.java | 105 +++++-
.../security/NMTokenSecretManagerInNM.java | 5 +-
.../recovery/NMMemoryStateStoreService.java | 55 ++-
.../TestNMLeveldbStateStoreService.java | 96 ++++-
.../TestNMContainerTokenSecretManager.java | 144 ++++++++
.../security/TestNMTokenSecretManagerInNM.java | 8 +-
.../scheduler/fair/FairScheduler.java | 15 +-
.../fair/FairSchedulerConfiguration.java | 9 +
.../security/DelegationTokenRenewer.java | 6 +-
.../fair/TestFairSchedulerPreemption.java | 2 +-
.../security/TestDelegationTokenRenewer.java | 36 +-
.../src/site/apt/FairScheduler.apt.vm | 6 +
.../src/site/apt/ResourceManagerRest.apt.vm | 105 +++---
58 files changed, 1789 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] git commit: MAPREDUCE-6025: distribute nativetask.so
to right folder (Manu Zhang)
Posted by ar...@apache.org.
MAPREDUCE-6025: distribute nativetask.so to right folder (Manu Zhang)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1618120 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d39367f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d39367f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d39367f
Branch: refs/heads/HDFS-6581
Commit: 6d39367f6a95383b634d79b6df35a278e0d89fe0
Parents: eace4dd
Author: Sean Zhong <se...@apache.org>
Authored: Fri Aug 15 08:55:49 2014 +0000
Committer: Sean Zhong <se...@apache.org>
Committed: Fri Aug 15 08:55:49 2014 +0000
----------------------------------------------------------------------
hadoop-dist/pom.xml | 1 -
hadoop-mapreduce-project/pom.xml | 28 ++++++++++++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d39367f/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 62a27f4..149f404 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -122,7 +122,6 @@
run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version}/* .
run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
- run cp -r $ROOT/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target/hadoop-mapreduce-client-nativetask-${project.version}/* .
run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* .
echo
echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d39367f/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 772a2a3..9110942 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -236,6 +236,34 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo file="${project.build.directory}/dist-copynativelibs.sh">
+ LIB_DIR="${mr.basedir}/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target/native/target/usr/local/lib"
+ if [ -d $${LIB_DIR} ] ; then
+ TARGET_DIR="${project.build.directory}/${project.artifactId}-${project.version}/lib/native"
+ mkdir -p $${TARGET_DIR}
+ cp -R $${LIB_DIR}/lib* $${TARGET_DIR}
+ fi
+ </echo>
+ <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <arg line="./dist-copynativelibs.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<dependencies>
<dependency>
[11/50] [abbrv] git commit: MAPREDUCE-6000. native-task: Simplify
ByteBufferDataReader/Writer. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613036 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66351803
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66351803
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66351803
Branch: refs/heads/HDFS-6581
Commit: 663518032449940f136ae8a9f149ed453837a8d8
Parents: 77acc70
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jul 24 08:24:05 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jul 24 08:24:05 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../nativetask/buffer/ByteBufferDataReader.java | 126 +----------------
.../nativetask/buffer/ByteBufferDataWriter.java | 141 ++++---------------
.../buffer/TestByteBufferReadWrite.java | 93 +++++-------
4 files changed, 75 insertions(+), 286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index e12f743..aa695cf 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -5,3 +5,4 @@ MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd)
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
+MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
index 5af7180..24f402d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapred.nativetask.buffer;
+import com.google.common.base.Charsets;
+
import java.io.DataInput;
import java.io.EOFException;
import java.io.IOException;
@@ -31,11 +33,13 @@ import java.nio.ByteBuffer;
public class ByteBufferDataReader extends DataInputStream {
private ByteBuffer byteBuffer;
private char lineCache[];
+ private java.io.DataInputStream javaReader;
public ByteBufferDataReader(InputBuffer buffer) {
if (buffer != null) {
- this.byteBuffer = buffer.getByteBuffer();
+ reset(buffer);
}
+ javaReader = new java.io.DataInputStream(this);
}
public void reset(InputBuffer buffer) {
@@ -128,128 +132,12 @@ public class ByteBufferDataReader extends DataInputStream {
@Override
public String readLine() throws IOException {
-
- InputStream in = this;
-
- char buf[] = lineCache;
-
- if (buf == null) {
- buf = lineCache = new char[128];
- }
-
- int room = buf.length;
- int offset = 0;
- int c;
-
- loop: while (true) {
- switch (c = in.read()) {
- case -1:
- case '\n':
- break loop;
-
- case '\r':
- final int c2 = in.read();
- if ((c2 != '\n') && (c2 != -1)) {
- if (!(in instanceof PushbackInputStream)) {
- in = new PushbackInputStream(in);
- }
- ((PushbackInputStream) in).unread(c2);
- }
- break loop;
-
- default:
- if (--room < 0) {
- buf = new char[offset + 128];
- room = buf.length - offset - 1;
- System.arraycopy(lineCache, 0, buf, 0, offset);
- lineCache = buf;
- }
- buf[offset++] = (char) c;
- break;
- }
- }
- if ((c == -1) && (offset == 0)) {
- return null;
- }
- return String.copyValueOf(buf, 0, offset);
+ return javaReader.readLine();
}
@Override
public final String readUTF() throws IOException {
- return readUTF(this);
- }
-
- private final static String readUTF(DataInput in) throws IOException {
- final int utflen = in.readUnsignedShort();
- byte[] bytearr = null;
- char[] chararr = null;
-
- bytearr = new byte[utflen];
- chararr = new char[utflen];
-
- int c, char2, char3;
- int count = 0;
- int chararr_count = 0;
-
- in.readFully(bytearr, 0, utflen);
-
- while (count < utflen) {
- c = bytearr[count] & 0xff;
- if (c > 127) {
- break;
- }
- count++;
- chararr[chararr_count++] = (char) c;
- }
-
- while (count < utflen) {
- c = bytearr[count] & 0xff;
- switch (c >> 4) {
- case 0:
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- case 6:
- case 7:
- /* 0xxxxxxx */
- count++;
- chararr[chararr_count++] = (char) c;
- break;
- case 12:
- case 13:
- /* 110x xxxx 10xx xxxx */
- count += 2;
- if (count > utflen) {
- throw new UTFDataFormatException("malformed input: partial character at end");
- }
- char2 = bytearr[count - 1];
- if ((char2 & 0xC0) != 0x80) {
- throw new UTFDataFormatException("malformed input around byte " + count);
- }
- chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
- break;
- case 14:
- /* 1110 xxxx 10xx xxxx 10xx xxxx */
- count += 3;
- if (count > utflen) {
- throw new UTFDataFormatException("malformed input: partial character at end");
- }
- char2 = bytearr[count - 2];
- char3 = bytearr[count - 1];
- if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) {
- throw new UTFDataFormatException("malformed input around byte " + (count - 1));
- }
- chararr[chararr_count++] = (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
- break;
- default:
- /* 10xx xxxx, 1111 xxxx */
- throw new UTFDataFormatException("malformed input around byte " + count);
- }
- }
- // The number of chars produced may be less than utflen
- return new String(chararr, 0, chararr_count);
+ return javaReader.readUTF();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
index 36b2fcf..aa87e0d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
@@ -22,31 +22,42 @@ import java.io.IOException;
import java.io.UTFDataFormatException;
import java.nio.ByteBuffer;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.UnsignedInteger;
+import com.google.common.primitives.UnsignedInts;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
/**
- * write data to a output buffer
+ * DataOutputStream implementation which buffers data in a fixed-size
+ * ByteBuffer.
+ * When the byte buffer has filled up, synchronously passes the buffer
+ * to a downstream NativeDataTarget.
*/
public class ByteBufferDataWriter extends DataOutputStream {
- private ByteBuffer buffer;
+ private final ByteBuffer buffer;
private final NativeDataTarget target;
- private void checkSizeAndFlushNecessary(int length) throws IOException {
+ private final static byte TRUE = (byte) 1;
+ private final static byte FALSE = (byte) 0;
+ private final java.io.DataOutputStream javaWriter;
+
+ private void checkSizeAndFlushIfNecessary(int length) throws IOException {
if (buffer.position() > 0 && buffer.remaining() < length) {
flush();
}
}
public ByteBufferDataWriter(NativeDataTarget handler) {
- if (null != handler) {
- this.buffer = handler.getOutputBuffer().getByteBuffer();
- }
+ Preconditions.checkNotNull(handler);
+ this.buffer = handler.getOutputBuffer().getByteBuffer();
this.target = handler;
+ this.javaWriter = new java.io.DataOutputStream(this);
}
@Override
public synchronized void write(int v) throws IOException {
- checkSizeAndFlushNecessary(1);
+ checkSizeAndFlushIfNecessary(1);
buffer.put((byte) v);
}
@@ -89,164 +100,72 @@ public class ByteBufferDataWriter extends DataOutputStream {
target.finishSendData();
}
- private final static byte TRUE = (byte) 1;
- private final static byte FALSE = (byte) 0;
-
@Override
public final void writeBoolean(boolean v) throws IOException {
- checkSizeAndFlushNecessary(1);
+ checkSizeAndFlushIfNecessary(1);
buffer.put(v ? TRUE : FALSE);
}
@Override
public final void writeByte(int v) throws IOException {
- checkSizeAndFlushNecessary(1);
+ checkSizeAndFlushIfNecessary(1);
buffer.put((byte) v);
}
@Override
public final void writeShort(int v) throws IOException {
- checkSizeAndFlushNecessary(2);
+ checkSizeAndFlushIfNecessary(2);
buffer.putShort((short) v);
}
@Override
public final void writeChar(int v) throws IOException {
- checkSizeAndFlushNecessary(2);
+ checkSizeAndFlushIfNecessary(2);
buffer.put((byte) ((v >>> 8) & 0xFF));
buffer.put((byte) ((v >>> 0) & 0xFF));
}
@Override
public final void writeInt(int v) throws IOException {
- checkSizeAndFlushNecessary(4);
+ checkSizeAndFlushIfNecessary(4);
buffer.putInt(v);
}
@Override
public final void writeLong(long v) throws IOException {
- checkSizeAndFlushNecessary(8);
+ checkSizeAndFlushIfNecessary(8);
buffer.putLong(v);
}
@Override
public final void writeFloat(float v) throws IOException {
- checkSizeAndFlushNecessary(4);
+ checkSizeAndFlushIfNecessary(4);
writeInt(Float.floatToIntBits(v));
}
@Override
public final void writeDouble(double v) throws IOException {
- checkSizeAndFlushNecessary(8);
+ checkSizeAndFlushIfNecessary(8);
writeLong(Double.doubleToLongBits(v));
}
@Override
public final void writeBytes(String s) throws IOException {
- final int len = s.length();
-
- int remain = len;
- int offset = 0;
- while (remain > 0) {
- int currentFlush = 0;
- if (buffer.remaining() > 0) {
- currentFlush = Math.min(buffer.remaining(), remain);
-
- for (int i = 0; i < currentFlush; i++) {
- buffer.put((byte) s.charAt(offset + i));
- }
-
- remain -= currentFlush;
- offset += currentFlush;
- } else {
- flush();
- }
- }
+ javaWriter.writeBytes(s);
}
@Override
public final void writeChars(String s) throws IOException {
- final int len = s.length();
-
- int remain = len;
- int offset = 0;
-
- while (remain > 0) {
- int currentFlush = 0;
- if (buffer.remaining() > 2) {
- currentFlush = Math.min(buffer.remaining() / 2, remain);
-
- for (int i = 0; i < currentFlush; i++) {
- buffer.putChar(s.charAt(offset + i));
- }
-
- remain -= currentFlush;
- offset += currentFlush;
- } else {
- flush();
- }
- }
+ javaWriter.writeChars(s);
}
@Override
public final void writeUTF(String str) throws IOException {
- writeUTF(str, this);
- }
-
- private int writeUTF(String str, DataOutput out) throws IOException {
- final int strlen = str.length();
- int utflen = 0;
- int c, count = 0;
-
- /* use charAt instead of copying String to char array */
- for (int i = 0; i < strlen; i++) {
- c = str.charAt(i);
- if ((c >= 0x0001) && (c <= 0x007F)) {
- utflen++;
- } else if (c > 0x07FF) {
- utflen += 3;
- } else {
- utflen += 2;
- }
- }
-
- if (utflen > 65535) {
- throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
- }
-
- final byte[] bytearr = new byte[utflen + 2];
- bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF);
- bytearr[count++] = (byte) ((utflen >>> 0) & 0xFF);
-
- int i = 0;
- for (i = 0; i < strlen; i++) {
- c = str.charAt(i);
- if (!((c >= 0x0001) && (c <= 0x007F))) {
- break;
- }
- bytearr[count++] = (byte) c;
- }
-
- for (; i < strlen; i++) {
- c = str.charAt(i);
- if ((c >= 0x0001) && (c <= 0x007F)) {
- bytearr[count++] = (byte) c;
-
- } else if (c > 0x07FF) {
- bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
- bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
- bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
- } else {
- bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
- bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
- }
- }
- write(bytearr, 0, utflen + 2);
- return utflen + 2;
+ javaWriter.writeUTF(str);
}
@Override
public boolean hasUnFlushedData() {
- return !(buffer.position() == 0);
+ return buffer.position() > 0;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
index 424354b..a405634 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestByteBufferReadWrite.java
@@ -17,20 +17,20 @@
*/
package org.apache.hadoop.mapred.nativetask.buffer;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
+import java.io.*;
+import com.google.common.base.Charsets;
+import com.google.common.primitives.Shorts;
import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
import junit.framework.Assert;
import junit.framework.TestCase;
+import org.mockito.Mockito;
-public class TestByteBufferReadWrite extends TestCase{
-
-
+public class TestByteBufferReadWrite extends TestCase {
public void testReadWrite() throws IOException {
byte[] buff = new byte[10000];
-
+
InputBuffer input = new InputBuffer(buff);
MockDataTarget target = new MockDataTarget(buff);
ByteBufferDataWriter writer = new ByteBufferDataWriter(target);
@@ -48,7 +48,7 @@ public class TestByteBufferReadWrite extends TestCase{
writer.writeBytes("goodboy");
writer.writeChars("hello");
writer.writeUTF("native task");
-
+
int length = target.getOutputBuffer().length();
input.rewind(0, length);
ByteBufferDataReader reader = new ByteBufferDataReader(input);
@@ -84,7 +84,29 @@ public class TestByteBufferReadWrite extends TestCase{
Assert.assertEquals(0, input.remaining());
}
-
+
+ /**
+ * Test that Unicode characters outside the basic multilingual plane,
+ * such as this cat face, are properly encoded.
+ */
+ public void testCatFace() throws IOException {
+ byte[] buff = new byte[10];
+ MockDataTarget target = new MockDataTarget(buff);
+ ByteBufferDataWriter writer = new ByteBufferDataWriter(target);
+ String catFace = "\uD83D\uDE38";
+ writer.writeUTF(catFace);
+
+ // Check that our own decoder can read it
+ InputBuffer input = new InputBuffer(buff);
+ input.rewind(0, buff.length);
+ ByteBufferDataReader reader = new ByteBufferDataReader(input);
+ assertEquals(catFace, reader.readUTF());
+
+ // Check that the standard Java one can read it too
+ String fromJava = new java.io.DataInputStream(new ByteArrayInputStream(buff)).readUTF();
+ assertEquals(catFace, fromJava);
+ }
+
public void testShortOfSpace() throws IOException {
byte[] buff = new byte[10];
MockDataTarget target = new MockDataTarget(buff);
@@ -100,20 +122,8 @@ public class TestByteBufferReadWrite extends TestCase{
public void testFlush() throws IOException {
byte[] buff = new byte[10];
- final Counter flushCount = new Counter();
- final Flag finishFlag = new Flag();
- MockDataTarget target = new MockDataTarget(buff) {
- @Override
- public void sendData() throws IOException {
- flushCount.increase();
- }
-
- @Override
- public void finishSendData() throws IOException {
- finishFlag.set(true);
- }
- };
-
+ MockDataTarget target = Mockito.spy(new MockDataTarget(buff));
+
ByteBufferDataWriter writer = new ByteBufferDataWriter(target);
Assert.assertEquals(false, writer.hasUnFlushedData());
@@ -121,10 +131,9 @@ public class TestByteBufferReadWrite extends TestCase{
writer.write(new byte[100]);
Assert.assertEquals(true, writer.hasUnFlushedData());
- writer.close();
- Assert.assertEquals(11, flushCount.get());
- Assert.assertEquals(true, finishFlag.get());
-
+ writer.close();
+ Mockito.verify(target, Mockito.times(11)).sendData();
+ Mockito.verify(target).finishSendData();
}
private static String toString(byte[] str) throws UnsupportedEncodingException {
@@ -140,42 +149,14 @@ public class TestByteBufferReadWrite extends TestCase{
}
@Override
- public void sendData() throws IOException {
-
- }
+ public void sendData() throws IOException {}
@Override
- public void finishSendData() throws IOException {
-
- }
+ public void finishSendData() throws IOException {}
@Override
public OutputBuffer getOutputBuffer() {
return out;
}
}
-
- private static class Counter {
- private int count;
-
- public int get() {
- return count;
- }
-
- public void increase() {
- count++;
- }
- }
-
- private static class Flag {
- private boolean value;
-
- public void set(boolean status) {
- this.value = status;
- }
-
- public boolean get() {
- return this.value;
- }
- }
}
[15/50] [abbrv] git commit: MAPREDUCE-6005. native-task: Fix some
valgrind errors (Binglin Chang)
Posted by ar...@apache.org.
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1615489 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83a39673
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83a39673
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83a39673
Branch: refs/heads/HDFS-6581
Commit: 83a396733ea43564beee928f41041a341e26db05
Parents: 78d86a9
Author: Binglin Chang <bc...@apache.org>
Authored: Mon Aug 4 06:19:51 2014 +0000
Committer: Binglin Chang <bc...@apache.org>
Committed: Mon Aug 4 06:19:51 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../src/main/native/src/codec/GzipCodec.cc | 11 +---
.../src/main/native/src/lib/Buffers.cc | 2 +-
.../src/main/native/src/lib/FileSystem.cc | 54 +++++++-------------
.../src/main/native/src/lib/FileSystem.h | 5 --
.../src/main/native/src/lib/MemoryBlock.h | 4 ++
.../main/native/src/lib/NativeObjectFactory.cc | 2 +-
.../main/native/src/lib/NativeObjectFactory.h | 15 +++++-
.../native/src/lib/PartitionBucketIterator.cc | 7 ++-
.../src/main/native/src/util/StringUtil.cc | 15 ++----
.../src/main/native/src/util/StringUtil.h | 2 +-
.../src/main/native/src/util/WritableUtils.cc | 2 +-
.../src/main/native/test/TestCommand.cc | 12 ++---
.../src/main/native/test/TestCounter.cc | 15 +++++-
.../src/main/native/test/TestFileSystem.cc | 31 -----------
.../src/main/native/test/TestIFile.cc | 9 ++--
.../src/main/native/test/TestMain.cc | 8 ++-
.../src/main/native/test/lib/TestByteBuffer.cc | 1 +
.../lib/TestComparatorForDualPivotQuickSort.cc | 1 +
.../native/test/lib/TestComparatorForStdSort.cc | 1 +
.../native/test/lib/TestFixSizeContainer.cc | 3 ++
.../src/main/native/test/lib/TestIterator.cc | 40 +++++++--------
.../src/main/native/test/lib/TestKVBuffer.cc | 1 +
.../native/test/lib/TestMemBlockIterator.cc | 8 +++
.../src/main/native/test/lib/TestMemoryBlock.cc | 3 ++
.../main/native/test/lib/TestPartitionBucket.cc | 16 +++++-
.../src/main/native/test/lib/TestReadBuffer.cc | 4 +-
.../main/native/test/lib/TestReadWriteBuffer.cc | 4 +-
.../src/main/native/test/util/TestStringUtil.cc | 5 ++
29 files changed, 146 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 42e0af3..c70df12 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -8,3 +8,4 @@ MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
+MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
index 79eac8b..bb699e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
@@ -28,7 +28,6 @@ GzipCompressStream::GzipCompressStream(OutputStream * stream, uint32_t bufferSiz
: CompressStream(stream), _compressedBytesWritten(0), _zstream(NULL), _finished(false) {
_buffer = new char[bufferSizeHint];
_capacity = bufferSizeHint;
- std::cout << "gzip capacity " << _capacity << std::endl;
_zstream = malloc(sizeof(z_stream));
z_stream * zstream = (z_stream*)_zstream;
memset(zstream, 0, sizeof(z_stream));
@@ -44,6 +43,7 @@ GzipCompressStream::GzipCompressStream(OutputStream * stream, uint32_t bufferSiz
GzipCompressStream::~GzipCompressStream() {
if (_zstream != NULL) {
+ deflateEnd((z_stream*)_zstream);
free(_zstream);
_zstream = NULL;
}
@@ -52,17 +52,13 @@ GzipCompressStream::~GzipCompressStream() {
}
void GzipCompressStream::write(const void * buff, uint32_t length) {
- std::cout << "gzip " << length << std::endl;
z_stream * zstream = (z_stream*)_zstream;
zstream->next_in = (Bytef*)buff;
zstream->avail_in = length;
while (true) {
int ret = deflate(zstream, Z_NO_FLUSH);
- std::cout << "gzip ret status " << ret << std::endl;
if (ret == Z_OK) {
- std::cout << "gzip avail_out " << zstream->avail_out << std::endl;
if (zstream->avail_out == 0) {
- std::cout << "gzip write capacity " << _capacity << std::endl;
_stream->write(_buffer, _capacity);
_compressedBytesWritten += _capacity;
zstream->next_out = (Bytef *)_buffer;
@@ -79,7 +75,6 @@ void GzipCompressStream::write(const void * buff, uint32_t length) {
}
void GzipCompressStream::flush() {
- std::cout << "gzip flush called";
z_stream * zstream = (z_stream*)_zstream;
while (true) {
int ret = deflate(zstream, Z_FINISH);
@@ -111,7 +106,6 @@ void GzipCompressStream::resetState() {
}
void GzipCompressStream::close() {
- std::cout << "gzip close called";
if (!_finished) {
flush();
}
@@ -146,6 +140,7 @@ GzipDecompressStream::GzipDecompressStream(InputStream * stream, uint32_t buffer
GzipDecompressStream::~GzipDecompressStream() {
if (_zstream != NULL) {
+ inflateEnd((z_stream*)_zstream);
free(_zstream);
_zstream = NULL;
}
@@ -173,11 +168,9 @@ int32_t GzipDecompressStream::read(void * buff, uint32_t length) {
int ret = inflate(zstream, Z_NO_FLUSH);
if (ret == Z_OK || ret == Z_STREAM_END) {
if (zstream->avail_out == 0) {
-// printf("return %d\n", length);
return length;
}
} else {
-// printf("Error: %d\n", ret);
return -1;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
index 243b34e..d07f2fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
@@ -157,7 +157,7 @@ char * ReadBuffer::fillGet(uint32_t count) {
_capacity = newcap;
} else {
if (_remain > 0) {
- memcpy(_buff, current(), _remain);
+ memmove(_buff, current(), _remain);
}
}
_size = _remain;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
index 7888aa1..e137883 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
@@ -17,9 +17,9 @@
*/
#include <errno.h>
+#include <fcntl.h>
#include <dirent.h>
#include <sys/stat.h>
-#include <jni.h>
#include "commons.h"
#include "util/StringUtil.h"
#include "jniutils.h"
@@ -34,13 +34,12 @@ namespace NativeTask {
/////////////////////////////////////////////////////////////
FileInputStream::FileInputStream(const string & path) {
- _handle = fopen(path.c_str(), "rb");
- if (_handle != NULL) {
- _fd = fileno(_handle);
+ _fd = ::open(path.c_str(), O_RDONLY);
+ if (_fd >= 0) {
_path = path;
} else {
_fd = -1;
- THROW_EXCEPTION_EX(IOException, "Can't open raw file: [%s]", path.c_str());
+ THROW_EXCEPTION_EX(IOException, "Can't open file for read: [%s]", path.c_str());
}
_bytesRead = NativeObjectFactory::GetCounter(TaskCounters::FILESYSTEM_COUNTER_GROUP,
TaskCounters::FILE_BYTES_READ);
@@ -67,9 +66,8 @@ int32_t FileInputStream::read(void * buff, uint32_t length) {
}
void FileInputStream::close() {
- if (_handle != NULL) {
- fclose(_handle);
- _handle = NULL;
+ if (_fd >= 0) {
+ ::close(_fd);
_fd = -1;
}
}
@@ -77,13 +75,20 @@ void FileInputStream::close() {
/////////////////////////////////////////////////////////////
FileOutputStream::FileOutputStream(const string & path, bool overwite) {
- _handle = fopen(path.c_str(), "wb");
- if (_handle != NULL) {
- _fd = fileno(_handle);
+ int flags = 0;
+ if (overwite) {
+ flags = O_WRONLY | O_CREAT | O_TRUNC;
+ } else {
+ flags = O_WRONLY | O_CREAT | O_EXCL;
+ }
+ mode_t mask = umask(0);
+ umask(mask);
+ _fd = ::open(path.c_str(), flags, (0666 & ~mask));
+ if (_fd >= 0) {
_path = path;
} else {
_fd = -1;
- THROW_EXCEPTION_EX(IOException, "Open raw file failed: [%s]", path.c_str());
+ THROW_EXCEPTION_EX(IOException, "Can't open file for write: [%s]", path.c_str());
}
_bytesWrite = NativeObjectFactory::GetCounter(TaskCounters::FILESYSTEM_COUNTER_GROUP,
TaskCounters::FILE_BYTES_WRITTEN);
@@ -108,9 +113,8 @@ void FileOutputStream::flush() {
}
void FileOutputStream::close() {
- if (_handle != NULL) {
- fclose(_handle);
- _handle = NULL;
+ if (_fd >= 0) {
+ ::close(_fd);
_fd = -1;
}
}
@@ -251,28 +255,8 @@ extern RawFileSystem RawFileSystemInstance;
RawFileSystem RawFileSystemInstance = RawFileSystem();
-string FileSystem::getDefaultFsUri(Config * config) {
- const char * nm = config->get(FS_DEFAULT_NAME);
- if (nm == NULL) {
- nm = config->get("fs.defaultFS");
- }
- if (nm == NULL) {
- return string("file:///");
- } else {
- return string(nm);
- }
-}
-
FileSystem & FileSystem::getLocal() {
return RawFileSystemInstance;
}
-
-FileSystem & FileSystem::get(Config * config) {
- string uri = getDefaultFsUri(config);
- if (uri == "file:///") {
- return RawFileSystemInstance;
- }
-}
-
} // namespace Hadoap
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
index 4b501fa..7649947 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
@@ -34,7 +34,6 @@ class FileSystem;
class FileInputStream : public InputStream {
private:
string _path;
- FILE * _handle;
int _fd;
Counter * _bytesRead;
public:
@@ -57,7 +56,6 @@ public:
class FileOutputStream : public OutputStream {
private:
string _path;
- FILE * _handle;
int _fd;
Counter * _bytesWrite;
public:
@@ -117,10 +115,7 @@ public:
virtual void mkdirs(const string & path) {
}
- static string getDefaultFsUri(Config * config);
static FileSystem & getLocal();
- static FileSystem & getJava(Config * config);
- static FileSystem & get(Config * config);
};
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.h
index e97d98a..f85c3b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.h
@@ -69,6 +69,10 @@ private:
public:
MemoryBlock(char * pos, uint32_t size);
+ char * base() {
+ return _base;
+ }
+
bool sorted() {
return _sorted;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
index bcd00f4..ce3799d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
@@ -78,7 +78,7 @@ Config * NativeObjectFactory::GlobalConfig = &G_CONFIG;
float NativeObjectFactory::LastProgress = 0;
Progress * NativeObjectFactory::TaskProgress = NULL;
string NativeObjectFactory::LastStatus;
-set<Counter *> NativeObjectFactory::CounterSet;
+set<Counter *, CounterPtrCompare> NativeObjectFactory::CounterSet;
vector<Counter *> NativeObjectFactory::Counters;
vector<uint64_t> NativeObjectFactory::CounterLastUpdateValues;
bool NativeObjectFactory::Inited = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.h
index afbae2c..9a9d41a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.h
@@ -36,6 +36,19 @@ using std::pair;
class NativeLibrary;
+class CounterPtrCompare {
+public:
+ bool operator()(const Counter * lhs, const Counter * rhs) const {
+ if (lhs->group() < rhs->group()) {
+ return true;
+ } else if (lhs->group() == rhs->group()) {
+ return lhs->name() < rhs->name();
+ } else {
+ return false;
+ }
+ }
+};
+
/**
* Native object factory
*/
@@ -47,7 +60,7 @@ private:
static float LastProgress;
static Progress * TaskProgress;
static string LastStatus;
- static set<Counter *> CounterSet;
+ static set<Counter *, CounterPtrCompare> CounterSet;
static vector<Counter *> Counters;
static vector<uint64_t> CounterLastUpdateValues;
static bool Inited;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
index 4a2ca5d..0b26f21 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
@@ -43,9 +43,11 @@ PartitionBucketIterator::PartitionBucketIterator(PartitionBucket * pb, Comparato
MemBlockIteratorPtr blockIterator = new MemBlockIterator(block);
if (blockIterator->next()) {
_heap.push_back(blockIterator);
+ } else {
+ delete blockIterator;
}
}
- if (_heap.size() > 0) {
+ if (_heap.size() > 1) {
makeHeap(&(_heap[0]), &(_heap[0]) + _heap.size(), _comparator);
}
}
@@ -78,6 +80,9 @@ bool PartitionBucketIterator::next() {
heapify(base, 1, cur_heap_size, _comparator);
}
} else { // no more, pop heap
+ // after popHeap, the first element of heap will be removed
+ // and replaced by other element, so it needs to be deleted
+ delete _heap[0];
MemBlockIteratorPtr * base = &(_heap[0]);
popHeap(base, base + cur_heap_size, _comparator);
_heap.pop_back();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
index d70dcc8..4e5b94a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
@@ -61,22 +61,17 @@ string StringUtil::ToString(bool v) {
}
string StringUtil::ToString(float v) {
- char tmp[32];
- snprintf(tmp, 32, "%f", v);
- return tmp;
+ return Format("%f", v);
}
string StringUtil::ToString(double v) {
- char tmp[32];
- snprintf(tmp, 32, "%lf", v);
- return tmp;
+ return Format("%lf", v);
}
-string StringUtil::ToString(const void * v, uint32_t len) {
+string StringUtil::ToHexString(const void * v, uint32_t len) {
string ret = string(len * 2, '0');
for (uint32_t i = 0; i < len; i++) {
- ret[i] = (((uint8_t*)v)[i] >> 4) + '0';
- ret[i] = (((uint8_t*)v)[i] & 0xff) + '0';
+ snprintf(&(ret[i*2]), 3, "%02x", ((char*)v)[i]);
}
return ret;
}
@@ -110,7 +105,7 @@ string StringUtil::Format(const char * fmt, ...) {
len = vsnprintf(destbuff, len + 1, fmt, al);
va_end(al);
dest.append(destbuff, len);
- delete destbuff;
+ delete [] destbuff;
} else {
dest.append(tmp, len);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.h
index 384c678..05904d4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.h
@@ -38,7 +38,7 @@ public:
static string ToString(bool v);
static string ToString(float v);
static string ToString(double v);
- static string ToString(const void * v, uint32_t len);
+ static string ToHexString(const void * v, uint32_t len);
static int64_t toInt(const string & str);
static bool toBool(const string & str);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
index 3e8043a..a16c654 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
@@ -297,7 +297,7 @@ void WritableUtils::toString(string & dest, KeyValueType type, const void * data
dest.append(StringUtil::ToString(*(double*)data));
break;
case MD5HashType:
- dest.append(StringUtil::ToString(data, length));
+ dest.append(StringUtil::ToHexString(data, length));
break;
default:
dest.append((const char*)data, length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
index 04e80fd..bcc3b3c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc
@@ -25,14 +25,14 @@
using namespace NativeTask;
TEST(Command, equals) {
- Command * cmd1 = new Command(100, "hello command");
- Command * cmd2 = new Command(100, "hello command 2");
+ Command cmd1(100, "hello command");
+ Command cmd2(100, "hello command 2");
- ASSERT_TRUE(cmd1->equals(*cmd2));
- ASSERT_TRUE(cmd2->equals(*cmd1));
- ASSERT_EQ(100, cmd1->id());
+ ASSERT_TRUE(cmd1.equals(cmd2));
+ ASSERT_TRUE(cmd2.equals(cmd1));
+ ASSERT_EQ(100, cmd1.id());
std::string helloCommand = "hello command";
- ASSERT_EQ(0, helloCommand.compare(cmd1->description()));
+ ASSERT_EQ(helloCommand, cmd1.description());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
index e8760ea..d4550bc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc
@@ -17,11 +17,12 @@
*/
#include "commons.h"
+#include "NativeObjectFactory.h"
#include "BufferStream.h"
#include "Buffers.h"
#include "test_commons.h"
-TEST(Counter, test) {
+TEST(Counter, Counter) {
Counter counter1("group", "key");
const string & group = counter1.group();
const string & name = counter1.name();
@@ -33,3 +34,15 @@ TEST(Counter, test) {
counter1.increase(100);
ASSERT_EQ(100, counter1.get());
}
+
+TEST(Counter, CounterSet) {
+ Counter * counter1 = NativeObjectFactory::GetCounter("group0", "name0");
+ ASSERT_EQ(string("group0"), counter1->group());
+ ASSERT_EQ(string("name0"), counter1->name());
+ counter1->increase(100);
+ ASSERT_EQ(100, counter1->get());
+ Counter * counter2 = NativeObjectFactory::GetCounter("group0", "name0");
+ Counter * counter3 = NativeObjectFactory::GetCounter("group0", "name1");
+ ASSERT_EQ(counter1, counter2);
+ ASSERT_NE(counter1, counter3);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
index c29c486..8b5a76c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc
@@ -48,34 +48,3 @@ TEST(FileSystem, RawFileSystem) {
ASSERT_FALSE(fs.exists(temppath));
}
-// This test needs java CLASSPATH env to run
-// Enable it manually only if some changes are made to FileSystem.h/cc
-//TEST(FileSystem, JavaFileSystem) {
-// FileSystem & fs = FileSystem::getJava(TestConfig);
-// fs.mkdirs("temp");
-// string temppath = "temp/data";
-// string content;
-// GenerateKVTextLength(content, 4111111, "word");
-// FileOutputStream * output = (FileOutputStream*)fs.create(temppath, true);
-// output->write(content.data(), content.length());
-// output->close();
-// delete output;
-// FileInputStream * input = (FileInputStream*)fs.open(temppath);
-// char buff[102400];
-// int64_t total = 0;
-// while(true) {
-// int rd = input->read(buff, 102400);
-// if (rd<=0) {
-// break;
-// }
-// ASSERT_EQ(content.substr(total, rd), string(buff,rd));
-// total+=rd;
-// }
-// ASSERT_EQ(content.length(), total);
-// delete input;
-// ASSERT_EQ(fs.getLength(temppath), content.length());
-// ASSERT_TRUE(fs.exists(temppath));
-// fs.remove("temp");
-// ASSERT_FALSE(fs.exists(temppath));
-//}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 02afbc0..07a1d63 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -67,6 +67,7 @@ void TestIFileReadWrite(KeyValueType kvtype, int partition, int size,
vector<pair<string, string> > readkvs;
readIFile(readkvs, outputpath, kvtype, info, codec);
LOG("read finished");
+ delete info;
ASSERT_EQ(kvs.size() * partition, readkvs.size());
for (int i = 0; i < partition; i++) {
vector<pair<string, string> > cur_part(readkvs.begin() + i * kvs.size(),
@@ -169,13 +170,13 @@ TEST(IFile, TestGlibCBug) {
int32_t expect[5] = {-1538241715, -1288088794, -192294464, 563552421, 1661521654};
LOG("TestGlibCBug %s", path.c_str());
- IFileSegment * segments = new IFileSegment [1];
+ IFileSegment * segments = new IFileSegment[1];
segments[0].realEndOffset = 10000000;
- SingleSpillInfo * info = new SingleSpillInfo(segments, 1, path, CHECKSUM_NONE,
+ SingleSpillInfo info(segments, 1, path, CHECKSUM_NONE,
IntType, TextType, "");
InputStream * fileOut = FileSystem::getLocal().open(path);
- IFileReader * reader = new IFileReader(fileOut, info, true);
+ IFileReader * reader = new IFileReader(fileOut, &info, true);
const char * key = NULL;
uint32_t length = 0;
@@ -187,4 +188,6 @@ TEST(IFile, TestGlibCBug) {
ASSERT_EQ(expect[index], realKey);
index++;
}
+ delete reader;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
index a9d5a06..c62232a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc
@@ -26,6 +26,7 @@
#include "commons.h"
#include "Buffers.h"
#include "FileSystem.h"
+#include "NativeObjectFactory.h"
#include "test_commons.h"
extern "C" {
@@ -75,6 +76,7 @@ int main(int argc, char ** argv) {
int skip = gen ? 2 : 1;
TestConfig.parse(argc - skip, (const char **)(newArgv + skip));
}
+ delete [] newArgv;
try {
if (gen == true) {
string type = TestConfig.get("generate.type", "word");
@@ -92,12 +94,16 @@ int main(int argc, char ** argv) {
fout->close();
delete fout;
}
+ NativeObjectFactory::Release();
return 0;
} else {
- return RUN_ALL_TESTS();
+ int ret = RUN_ALL_TESTS();
+ NativeObjectFactory::Release();
+ return ret;
}
} catch (std::exception & e) {
fprintf(stderr, "Exception: %s", e.what());
+ NativeObjectFactory::Release();
return 1;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
index 9e874c7..228cace 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
@@ -42,5 +42,6 @@ TEST(ByteBuffer, read) {
ASSERT_EQ(20, byteBuffer.limit());
ASSERT_EQ(10, byteBuffer.position());
+ delete [] buff;
}
} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
index cb195dd..9572d87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc
@@ -83,6 +83,7 @@ TEST(ComparatorForDualPivotQuickSort, compare) {
compareResult = -1;
ASSERT_EQ(-1, comparator((char * )kv1 - buff, (char * )kv2 - buff));
+ delete [] buff;
}
} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
index 80d1186..6dd97ba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc
@@ -83,6 +83,7 @@ TEST(ComparatorForStdSort, compare) {
compareResult = -1;
ASSERT_EQ(true, comparator((char * )kv1 - buff, (char * )kv2 - buff));
+ delete [] buff;
}
} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
index e25212e..cd8fb27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
@@ -47,6 +47,9 @@ TEST(FixSizeContainer, test) {
char * c = container->base() + i;
ASSERT_EQ(toBeFilled[i], *c);
}
+
+ delete [] bytes;
+ delete container;
}
} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
index 39bd0a6..6e79796 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
@@ -24,28 +24,26 @@
namespace NativeTask {
class MockIterator : public KVIterator {
- std::vector<std::pair<int, int> > * kvs;
+ std::vector<std::pair<int, int> > kvs;
uint32_t index;
uint32_t expectedKeyGroupNum;
std::map<int, int> expectkeyCountMap;
- char * buffer;
+ char buffer[8];
public:
MockIterator()
- : index(0), buffer(NULL) {
- buffer = new char[8];
- kvs = new std::vector<std::pair<int, int> >();
- kvs->push_back(std::pair<int, int>(10, 100));
-
- kvs->push_back(std::pair<int, int>(10, 100));
- kvs->push_back(std::pair<int, int>(10, 101));
- kvs->push_back(std::pair<int, int>(10, 102));
-
- kvs->push_back(std::pair<int, int>(20, 200));
- kvs->push_back(std::pair<int, int>(20, 201));
- kvs->push_back(std::pair<int, int>(20, 202));
- kvs->push_back(std::pair<int, int>(30, 302));
- kvs->push_back(std::pair<int, int>(40, 302));
+ : index(0) {
+ kvs.push_back(std::pair<int, int>(10, 100));
+
+ kvs.push_back(std::pair<int, int>(10, 100));
+ kvs.push_back(std::pair<int, int>(10, 101));
+ kvs.push_back(std::pair<int, int>(10, 102));
+
+ kvs.push_back(std::pair<int, int>(20, 200));
+ kvs.push_back(std::pair<int, int>(20, 201));
+ kvs.push_back(std::pair<int, int>(20, 202));
+ kvs.push_back(std::pair<int, int>(30, 302));
+ kvs.push_back(std::pair<int, int>(40, 302));
this->expectedKeyGroupNum = 4;
expectkeyCountMap[10] = 4;
@@ -55,8 +53,8 @@ public:
}
bool next(Buffer & key, Buffer & outValue) {
- if (index < kvs->size()) {
- std::pair<int, int> value = kvs->at(index);
+ if (index < kvs.size()) {
+ std::pair<int, int> value = kvs.at(index);
*((int *)buffer) = value.first;
*(((int *)buffer) + 1) = value.second;
key.reset(buffer, 4);
@@ -88,11 +86,9 @@ void TestKeyGroupIterator() {
uint32_t length = 0;
key = groupIterator->getKey(length);
int * keyPtr = (int *)key;
- std::cout << "new key group(key group hold kvs of same key): " << *keyPtr << std::endl;
const char * value = NULL;
while (NULL != (value = groupIterator->nextValue(length))) {
int * valuePtr = (int *)value;
- std::cout << "==== key: " << *keyPtr << "value: " << *valuePtr << std::endl;
if (actualKeyCount.find(*keyPtr) == actualKeyCount.end()) {
actualKeyCount[*keyPtr] = 0;
@@ -108,8 +104,8 @@ void TestKeyGroupIterator() {
uint32_t expectedCount = expectedKeyCountMap[key];
ASSERT_EQ(expectedCount, keyCountIter->second);
}
-
- std::cout << "Done!!!!!!! " << std::endl;
+ delete groupIterator;
+ delete iter;
}
TEST(Iterator, keyGroupIterator) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
index c3c5113..32b5891 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc
@@ -48,6 +48,7 @@ TEST(KVBuffer, test) {
ASSERT_EQ(8, kv1->headerLength());
ASSERT_EQ(strlen(KEY) + strlen(VALUE) + 8, kv1->lengthConvertEndium());
+ delete [] buff;
}
} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
index 458055e..87fba15 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
@@ -43,6 +43,7 @@ TEST(MemoryBlockIterator, test) {
ASSERT_EQ(block.getKVBuffer(keyCount), kv);
keyCount++;
}
+ delete [] bytes;
}
class MemoryBlockFactory {
@@ -102,6 +103,13 @@ TEST(MemoryBlockIterator, compare) {
iter2->next();
ASSERT_EQ(false, comparator(iter1, iter2));
+
+ delete iter2;
+ delete iter1;
+ delete [] block2->base();
+ delete [] block1->base();
+ delete block2;
+ delete block1;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
index 665e8be..dc60ac3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
@@ -47,6 +47,7 @@ TEST(MemoryBlock, test) {
ASSERT_EQ(BUFFER_LENGTH - 2 * KV_SIZE, block.remainSpace());
ASSERT_EQ(false, block.sorted());
+ delete [] bytes;
}
TEST(MemoryBlock, overflow) {
@@ -64,6 +65,7 @@ TEST(MemoryBlock, overflow) {
ASSERT_EQ(1, block.getKVCount());
ASSERT_EQ(BUFFER_LENGTH - KV_SIZE, block.remainSpace());
+ delete [] bytes;
}
TEST(MemoryBlock, sort) {
@@ -101,6 +103,7 @@ TEST(MemoryBlock, sort) {
ASSERT_EQ(small, block.getKVBuffer(0));
ASSERT_EQ(medium, block.getKVBuffer(1));
ASSERT_EQ(big, block.getKVBuffer(2));
+ delete [] bytes;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
index f60c210..a975243 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc
@@ -83,17 +83,28 @@ TEST(PartitionBucket, multipleMemoryBlock) {
const uint32_t KV_SIZE = 700;
const uint32_t SMALL_KV_SIZE = 100;
+ // To suppress valgrind error
+ // the allocated buffer needs to be initialized before
+ // create iterator on the PartitionBucker, because
+ // those memory will be compared when create minheap
KVBuffer * kv1 = bucket->allocateKVBuffer(KV_SIZE);
+ memset(kv1, 0, KV_SIZE);
KVBuffer * kv2 = bucket->allocateKVBuffer(SMALL_KV_SIZE);
+ memset(kv2, 0, SMALL_KV_SIZE);
KVBuffer * kv3 = bucket->allocateKVBuffer(KV_SIZE);
+ memset(kv3, 0, KV_SIZE);
ASSERT_EQ(3, bucket->getKVCount());
KVIterator * NULLPOINTER = 0;
- ASSERT_NE(NULLPOINTER, bucket->getIterator());
+ KVIterator * iter = bucket->getIterator();
+ ASSERT_NE(NULLPOINTER, iter);
+ delete iter;
ASSERT_EQ(2, bucket->getMemoryBlockCount());
bucket->reset();
- ASSERT_EQ(NULLPOINTER, bucket->getIterator());
+ iter = bucket->getIterator();
+ ASSERT_EQ(NULLPOINTER, iter);
+ delete iter;
ASSERT_EQ(0, bucket->getMemoryBlockCount());
delete bucket;
@@ -206,6 +217,7 @@ TEST(PartitionBucket, spill) {
ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, third->valueLength);
ASSERT_EQ(bswap(BIG), (*(uint32_t * )(third->getKey())));
+ delete [] buff;
delete bucket;
delete pool;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
index e5e39af..648fe80 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
@@ -24,7 +24,7 @@
TEST(Buffers, AppendRead) {
string codec = "";
vector<string> data;
- Generate(data, 1000000, "word");
+ Generate(data, 100000, "word");
string dest;
dest.reserve(64 * 1024 * 1024);
OutputStringStream outputStream = OutputStringStream(dest);
@@ -46,7 +46,7 @@ TEST(Buffers, AppendRead) {
TEST(Buffers, AppendReadSnappy) {
string codec = "org.apache.hadoop.io.compress.SnappyCodec";
vector<string> data;
- Generate(data, 1000000, "word");
+ Generate(data, 100000, "word");
string dest;
dest.reserve(64 * 1024 * 1024);
OutputStringStream outputStream = OutputStringStream(dest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
index 5ac4ec7..efbb8ea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc
@@ -42,21 +42,19 @@ TEST(ReadWriteBuffer, readAndWrite) {
}
uint32_t writePoint = buff.getWritePoint();
- LOG("Current Write Point: %d", writePoint);
for (int i = 0; i < REPEAT; i++) {
ASSERT_EQ(INT, buff.readInt());
ASSERT_EQ(LONG, buff.readLong());
string * read = buff.readString();
- LOG("READ STRING: %s", read->c_str());
ASSERT_EQ(0, STR.compare(read->c_str()));
delete read;
ASSERT_EQ(POINTER, buff.readPointer());
read = buff.readString();
- LOG("READ STRING: %s", read->c_str());
ASSERT_EQ(0, STR.compare(read->c_str()));
+ delete read;
}
uint32_t readPoint = buff.getReadPoint();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc
index d6dc93f..5cc79a9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc
@@ -27,6 +27,11 @@ TEST(StringUtil, Convertion) {
ASSERT_EQ(StringUtil::ToString(111, ' ', 40), " 111");
}
+TEST(StringUtil, ToHexString) {
+ uint8_t buff[4] = {'i', 'j', 'k', 'l'};
+ ASSERT_EQ(StringUtil::ToHexString(buff, 4), string("696a6b6c"));
+}
+
TEST(StringUtil, Format) {
string t = StringUtil::Format("%d %d %d %.3lf %s", 1, 2, 3, 1.333, "aaaaaaaaaaa");
ASSERT_EQ(t, "1 2 3 1.333 aaaaaaaaaaa");
[31/50] [abbrv] git commit: Merge remote-tracking branch
'origin/trunk' into MR-2841
Posted by ar...@apache.org.
Merge remote-tracking branch 'origin/trunk' into MR-2841
Conflicts:
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cce7d1e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cce7d1e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cce7d1e2
Branch: refs/heads/HDFS-6581
Commit: cce7d1e2f96ff75fe19ab2879ddd90a898ca5b18
Parents: 7ade9b0 727331b
Author: Todd Lipcon <to...@apache.org>
Authored: Tue Sep 2 15:55:54 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Sep 2 15:55:54 2014 -0700
----------------------------------------------------------------------
.gitignore | 1 +
BUILDING.txt | 22 +
dev-support/test-patch.sh | 26 +-
.../main/resources/assemblies/hadoop-dist.xml | 7 +
.../dev-support/findbugsExcludeFile.xml | 38 +
hadoop-common-project/hadoop-auth/pom.xml | 26 +
.../authentication/client/AuthenticatedURL.java | 36 +-
.../server/AuthenticationFilter.java | 109 +-
.../server/KerberosAuthenticationHandler.java | 21 +-
.../server/PseudoAuthenticationHandler.java | 21 +-
.../util/RandomSignerSecretProvider.java | 49 +
.../util/RolloverSignerSecretProvider.java | 139 ++
.../security/authentication/util/Signer.java | 46 +-
.../util/SignerSecretProvider.java | 62 +
.../util/StringSignerSecretProvider.java | 49 +
.../client/AuthenticatorTestCase.java | 137 +-
.../client/TestAuthenticatedURL.java | 38 +-
.../client/TestKerberosAuthenticator.java | 58 +-
.../server/TestAuthenticationFilter.java | 118 +-
.../util/TestRandomSignerSecretProvider.java | 63 +
.../util/TestRolloverSignerSecretProvider.java | 79 +
.../authentication/util/TestSigner.java | 85 +-
.../util/TestStringSignerSecretProvider.java | 33 +
hadoop-common-project/hadoop-common/CHANGES.txt | 502 +++++--
.../dev-support/findbugsExcludeFile.xml | 5 +
hadoop-common-project/hadoop-common/pom.xml | 34 +-
.../hadoop-common/src/CMakeLists.txt | 34 +
.../hadoop-common/src/JNIFlags.cmake | 6 +
.../hadoop-common/src/config.h.cmake | 1 +
.../src/contrib/bash-tab-completion/hadoop.sh | 28 +-
.../hadoop-common/src/main/bin/hadoop | 229 +--
.../src/main/bin/hadoop-config.cmd | 10 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 409 ++---
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 214 +--
.../src/main/bin/hadoop-daemons.sh | 37 +-
.../src/main/bin/hadoop-functions.sh | 1066 ++++++++++++++
.../src/main/bin/hadoop-layout.sh.example | 93 ++
.../hadoop-common/src/main/bin/hadoop.cmd | 33 +-
.../hadoop-common/src/main/bin/rcc | 52 +-
.../hadoop-common/src/main/bin/slaves.sh | 51 +-
.../hadoop-common/src/main/bin/start-all.sh | 38 +-
.../hadoop-common/src/main/bin/stop-all.sh | 36 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 432 +++++-
.../org/apache/hadoop/conf/Configuration.java | 78 +-
.../hadoop/conf/ReconfigurationServlet.java | 2 +
.../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 +
.../org/apache/hadoop/crypto/CipherSuite.java | 115 ++
.../org/apache/hadoop/crypto/CryptoCodec.java | 179 +++
.../apache/hadoop/crypto/CryptoInputStream.java | 680 +++++++++
.../hadoop/crypto/CryptoOutputStream.java | 280 ++++
.../apache/hadoop/crypto/CryptoStreamUtils.java | 70 +
.../org/apache/hadoop/crypto/Decryptor.java | 72 +
.../org/apache/hadoop/crypto/Encryptor.java | 71 +
.../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 +++
.../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 +++
.../org/apache/hadoop/crypto/OpensslCipher.java | 289 ++++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 248 +++-
.../apache/hadoop/crypto/key/KeyProvider.java | 32 +-
.../crypto/key/KeyProviderCryptoExtension.java | 91 +-
.../KeyProviderDelegationTokenExtension.java | 8 +-
.../hadoop/crypto/key/KeyProviderExtension.java | 1 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 89 +-
.../apache/hadoop/crypto/key/UserProvider.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 117 +-
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 2 +-
.../crypto/random/OpensslSecureRandom.java | 122 ++
.../hadoop/crypto/random/OsSecureRandom.java | 115 ++
.../apache/hadoop/fs/AbstractFileSystem.java | 103 +-
.../apache/hadoop/fs/ChecksumFileSystem.java | 8 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 8 +-
.../hadoop/fs/CommonConfigurationKeys.java | 4 +-
.../fs/CommonConfigurationKeysPublic.java | 34 +-
.../org/apache/hadoop/fs/ContentSummary.java | 46 +-
.../apache/hadoop/fs/FSDataOutputStream.java | 2 +-
.../org/apache/hadoop/fs/FSOutputSummer.java | 107 +-
.../java/org/apache/hadoop/fs/FileContext.java | 141 +-
.../apache/hadoop/fs/FileEncryptionInfo.java | 102 ++
.../java/org/apache/hadoop/fs/FileSystem.java | 159 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 7 +
.../java/org/apache/hadoop/fs/FilterFs.java | 8 +
.../main/java/org/apache/hadoop/fs/Globber.java | 8 +-
.../fs/crypto/CryptoFSDataInputStream.java | 37 +
.../fs/crypto/CryptoFSDataOutputStream.java | 47 +
.../hadoop/fs/shell/CommandWithDestination.java | 75 +-
.../apache/hadoop/fs/shell/CopyCommands.java | 6 +-
.../java/org/apache/hadoop/fs/shell/Count.java | 36 +-
.../java/org/apache/hadoop/fs/shell/Delete.java | 6 +-
.../org/apache/hadoop/fs/shell/Display.java | 12 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 8 +
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 7 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 10 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 9 +
.../org/apache/hadoop/http/HttpServer2.java | 16 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 9 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 19 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 8 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 522 +++++++
.../hadoop/ipc/DecayRpcSchedulerMXBean.java | 30 +
.../org/apache/hadoop/ipc/FairCallQueue.java | 449 ++++++
.../apache/hadoop/ipc/FairCallQueueMXBean.java | 27 +
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 20 +-
.../org/apache/hadoop/ipc/RpcMultiplexer.java | 32 +
.../org/apache/hadoop/ipc/RpcScheduler.java | 29 +
.../main/java/org/apache/hadoop/ipc/Server.java | 43 +-
.../ipc/WeightedRoundRobinMultiplexer.java | 2 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 18 +-
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 8 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 9 +-
.../metrics2/impl/MetricsCollectorImpl.java | 6 +-
.../apache/hadoop/metrics2/lib/MutableStat.java | 8 +
.../net/NetworkTopologyWithNodeGroup.java | 2 +-
.../JniBasedUnixGroupsMappingWithFallback.java | 3 +-
.../hadoop/security/LdapGroupsMapping.java | 26 +-
.../apache/hadoop/security/NetgroupCache.java | 17 +-
.../apache/hadoop/security/SecurityUtil.java | 3 +-
.../hadoop/security/WhitelistBasedResolver.java | 149 ++
.../hadoop/security/alias/CredentialShell.java | 65 +-
.../hadoop/security/alias/UserProvider.java | 5 -
.../authorize/ServiceAuthorizationManager.java | 38 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 22 +-
.../web/DelegationTokenAuthenticatedURL.java | 398 +++++
.../DelegationTokenAuthenticationFilter.java | 274 ++++
.../DelegationTokenAuthenticationHandler.java | 369 +++++
.../web/DelegationTokenAuthenticator.java | 261 ++++
.../web/DelegationTokenIdentifier.java | 64 +
.../delegation/web/DelegationTokenManager.java | 153 ++
.../web/HttpUserGroupInformation.java | 43 +
...rosDelegationTokenAuthenticationHandler.java | 54 +
.../KerberosDelegationTokenAuthenticator.java | 46 +
...udoDelegationTokenAuthenticationHandler.java | 55 +
.../web/PseudoDelegationTokenAuthenticator.java | 54 +
.../token/delegation/web/ServletUtils.java | 59 +
.../apache/hadoop/tracing/SpanReceiverHost.java | 153 ++
.../hadoop/util/ApplicationClassLoader.java | 219 +++
.../org/apache/hadoop/util/CacheableIPList.java | 76 +
.../java/org/apache/hadoop/util/Classpath.java | 125 ++
.../apache/hadoop/util/CombinedIPWhiteList.java | 60 +
.../org/apache/hadoop/util/DataChecksum.java | 22 +-
.../org/apache/hadoop/util/DiskChecker.java | 14 +
.../org/apache/hadoop/util/FileBasedIPList.java | 102 ++
.../hadoop/util/GenericOptionsParser.java | 6 +
.../java/org/apache/hadoop/util/IPList.java | 33 +
.../org/apache/hadoop/util/MachineList.java | 13 +-
.../apache/hadoop/util/NativeCodeLoader.java | 5 +
.../org/apache/hadoop/util/NativeCrc32.java | 43 +-
.../hadoop/util/NativeLibraryChecker.java | 21 +-
.../apache/hadoop/util/PerformanceAdvisory.java | 25 +
.../java/org/apache/hadoop/util/ProtoUtil.java | 11 +
.../java/org/apache/hadoop/util/RunJar.java | 115 +-
.../org/apache/hadoop/util/StringUtils.java | 14 +
.../java/org/apache/hadoop/util/ToolRunner.java | 2 +-
.../org/apache/hadoop/util/VersionInfo.java | 3 +-
.../hadoop-common/src/main/native/README | 7 +-
.../org/apache/hadoop/crypto/OpensslCipher.c | 382 +++++
.../hadoop/crypto/org_apache_hadoop_crypto.h | 61 +
.../hadoop/crypto/random/OpensslSecureRandom.c | 335 +++++
.../random/org_apache_hadoop_crypto_random.h | 40 +
.../org/apache/hadoop/io/nativeio/NativeIO.c | 72 +-
.../apache/hadoop/security/hadoop_user_info.c | 10 +
.../org/apache/hadoop/util/NativeCodeLoader.c | 10 +
.../src/org/apache/hadoop/util/NativeCrc32.c | 96 +-
.../src/org/apache/hadoop/util/bulk_crc32.c | 60 +-
.../src/org/apache/hadoop/util/bulk_crc32.h | 49 +-
.../org/apache/hadoop/util/test_bulk_crc32.c | 6 +-
.../src/main/proto/RpcHeader.proto | 13 +
.../src/main/resources/core-default.xml | 88 +-
.../src/site/apt/CommandsManual.apt.vm | 380 +----
.../src/site/apt/Compatibility.apt.vm | 9 +-
.../src/site/apt/FileSystemShell.apt.vm | 17 +-
.../src/site/apt/NativeLibraries.apt.vm | 22 +-
.../src/site/apt/ServiceLevelAuth.apt.vm | 21 +
.../hadoop-common/src/site/apt/Tracing.apt.vm | 169 +++
.../apache/hadoop/conf/TestConfiguration.java | 8 +
.../hadoop/crypto/CryptoStreamsTestBase.java | 721 +++++++++
.../apache/hadoop/crypto/TestCryptoCodec.java | 186 +++
.../apache/hadoop/crypto/TestCryptoStreams.java | 376 +++++
.../crypto/TestCryptoStreamsForLocalFS.java | 120 ++
.../hadoop/crypto/TestCryptoStreamsNormal.java | 123 ++
...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 31 +
.../apache/hadoop/crypto/TestOpensslCipher.java | 110 ++
.../crypto/key/TestCachingKeyProvider.java | 6 +
.../hadoop/crypto/key/TestKeyProvider.java | 17 +-
.../key/TestKeyProviderCryptoExtension.java | 11 +-
...TestKeyProviderDelegationTokenExtension.java | 13 +-
.../crypto/key/TestKeyProviderFactory.java | 79 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 56 +-
.../crypto/random/TestOpensslSecureRandom.java | 114 ++
.../crypto/random/TestOsSecureRandom.java | 139 ++
.../apache/hadoop/fs/TestContentSummary.java | 248 ++++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 3 +
.../org/apache/hadoop/fs/shell/TestCount.java | 270 ++++
.../apache/hadoop/fs/shell/TestTextCommand.java | 77 +-
.../org/apache/hadoop/http/TestHttpServer.java | 6 +-
.../hadoop/ipc/TestDecayRpcScheduler.java | 225 +++
.../apache/hadoop/ipc/TestFairCallQueue.java | 392 +++++
.../java/org/apache/hadoop/ipc/TestRPC.java | 8 +-
.../hadoop/security/TestLdapGroupsMapping.java | 58 +
.../hadoop/security/TestNetgroupCache.java | 127 ++
.../security/TestWhitelistBasedResolver.java | 163 ++
.../hadoop/security/alias/TestCredShell.java | 49 +-
.../authorize/TestServiceAuthorization.java | 117 ++
.../hadoop/security/ssl/KeyStoreTestUtil.java | 41 +
.../hadoop/security/ssl/TestSSLFactory.java | 58 +-
...tionTokenAuthenticationHandlerWithMocks.java | 364 +++++
.../web/TestDelegationTokenManager.java | 59 +
.../delegation/web/TestWebDelegationToken.java | 913 ++++++++++++
.../apache/hadoop/util/ClassLoaderCheck.java | 33 +
.../hadoop/util/ClassLoaderCheckMain.java | 34 +
.../hadoop/util/ClassLoaderCheckSecond.java | 24 +
.../hadoop/util/ClassLoaderCheckThird.java | 24 +
.../hadoop/util/TestApplicationClassLoader.java | 136 ++
.../apache/hadoop/util/TestCacheableIPList.java | 188 +++
.../org/apache/hadoop/util/TestClasspath.java | 176 +++
.../apache/hadoop/util/TestDataChecksum.java | 154 +-
.../apache/hadoop/util/TestFileBasedIPList.java | 215 +++
.../hadoop/util/TestGenericOptionsParser.java | 66 +
.../hadoop/util/TestNativeCodeLoader.java | 4 +
.../java/org/apache/hadoop/util/TestRunJar.java | 66 +-
.../src/test/resources/testConf.xml | 24 +-
hadoop-common-project/hadoop-kms/pom.xml | 1 -
.../hadoop/crypto/key/kms/server/KMS.java | 356 +++--
.../hadoop/crypto/key/kms/server/KMSACLs.java | 3 +-
.../hadoop/crypto/key/kms/server/KMSAudit.java | 192 ++-
.../key/kms/server/KMSAuthenticationFilter.java | 38 +-
.../crypto/key/kms/server/KMSConfiguration.java | 9 +-
.../key/kms/server/KMSExceptionsProvider.java | 16 +-
.../crypto/key/kms/server/KMSJMXServlet.java | 36 +
.../crypto/key/kms/server/KMSMDCFilter.java | 17 +-
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 20 +-
.../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 2 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 99 ++
.../hadoop/crypto/key/kms/server/TestKMS.java | 261 +++-
.../crypto/key/kms/server/TestKMSACLs.java | 10 +-
.../crypto/key/kms/server/TestKMSAudit.java | 135 ++
.../test/resources/log4j-kmsaudit.properties | 25 +
.../hadoop/nfs/nfs3/Nfs3FileAttributes.java | 14 +-
.../hadoop/nfs/nfs3/request/ACCESS3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/COMMIT3Request.java | 23 +-
.../hadoop/nfs/nfs3/request/CREATE3Request.java | 18 +-
.../hadoop/nfs/nfs3/request/FSINFO3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/FSSTAT3Request.java | 15 +-
.../nfs/nfs3/request/GETATTR3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/LINK3Request.java | 61 +
.../hadoop/nfs/nfs3/request/LOOKUP3Request.java | 9 +-
.../hadoop/nfs/nfs3/request/MKDIR3Request.java | 26 +-
.../hadoop/nfs/nfs3/request/MKNOD3Request.java | 89 ++
.../hadoop/nfs/nfs3/request/NFS3Request.java | 45 +
.../nfs/nfs3/request/PATHCONF3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/READ3Request.java | 9 +-
.../nfs/nfs3/request/READDIR3Request.java | 30 +-
.../nfs/nfs3/request/READDIRPLUS3Request.java | 33 +-
.../nfs/nfs3/request/READLINK3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/REMOVE3Request.java | 20 +-
.../hadoop/nfs/nfs3/request/RENAME3Request.java | 37 +-
.../hadoop/nfs/nfs3/request/RMDIR3Request.java | 20 +-
.../nfs/nfs3/request/RequestWithHandle.java | 16 +-
.../nfs/nfs3/request/SETATTR3Request.java | 29 +-
.../nfs/nfs3/request/SYMLINK3Request.java | 30 +-
.../hadoop/nfs/nfs3/request/SetAttr3.java | 9 +
.../hadoop/nfs/nfs3/request/WRITE3Request.java | 13 +-
.../hadoop/nfs/nfs3/response/LINK3Response.java | 54 +
.../nfs/nfs3/response/MKNOD3Response.java | 68 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
.../hadoop/fs/http/client/HttpFSFileSystem.java | 127 +-
.../client/HttpFSKerberosAuthenticator.java | 188 ---
.../http/client/HttpFSPseudoAuthenticator.java | 47 -
.../http/server/HttpFSAuthenticationFilter.java | 25 +-
.../HttpFSKerberosAuthenticationHandler.java | 230 ---
.../http/server/HttpFSParametersProvider.java | 108 +-
.../hadoop/fs/http/server/HttpFSServer.java | 144 +-
.../fs/http/server/HttpFSServerWebApp.java | 4 -
.../lib/service/DelegationTokenIdentifier.java | 61 -
.../lib/service/DelegationTokenManager.java | 78 -
.../DelegationTokenManagerException.java | 51 -
.../apache/hadoop/lib/service/ProxyUser.java | 31 -
.../security/DelegationTokenManagerService.java | 242 ---
.../lib/service/security/ProxyUserService.java | 179 ---
.../apache/hadoop/lib/wsrs/UserProvider.java | 109 --
.../src/main/resources/httpfs-default.xml | 15 +-
...KerberosAuthenticationHandlerForTesting.java | 6 +-
.../http/server/TestHttpFSCustomUserName.java | 94 --
...TestHttpFSKerberosAuthenticationHandler.java | 316 ----
.../hadoop/fs/http/server/TestHttpFSServer.java | 12 +-
.../fs/http/server/TestHttpFSWithKerberos.java | 6 +-
.../TestDelegationTokenManagerService.java | 89 --
.../service/security/TestProxyUserService.java | 226 ---
.../hadoop/lib/wsrs/TestUserProvider.java | 142 --
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 507 ++++---
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 593 ++++++++
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 77 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 385 ++++-
.../dev-support/findbugsExcludeFile.xml | 9 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 128 +-
.../hadoop-hdfs/src/CMakeLists.txt | 78 +-
.../bkjournal/BookKeeperEditLogInputStream.java | 5 +
.../src/main/bin/distribute-exclude.sh | 6 +-
.../hadoop-hdfs/src/main/bin/hdfs | 434 +++---
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 75 +-
.../src/main/bin/refresh-namenodes.sh | 40 +-
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 30 +-
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 142 +-
.../src/main/bin/start-secure-dns.sh | 34 +-
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 30 +-
.../hadoop-hdfs/src/main/bin/stop-dfs.sh | 105 +-
.../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 34 +-
.../src/main/conf/hadoop-metrics2.properties | 44 -
.../main/java/org/apache/hadoop/fs/Hdfs.java | 26 +-
.../main/java/org/apache/hadoop/fs/XAttr.java | 13 +-
.../apache/hadoop/hdfs/BlockReaderFactory.java | 27 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 207 ++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 21 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 220 +++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 92 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 77 +-
.../hadoop/hdfs/DistributedFileSystem.java | 81 +-
.../org/apache/hadoop/hdfs/StorageType.java | 10 +
.../hdfs/UnknownCipherSuiteException.java | 38 +
.../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 100 ++
.../hadoop/hdfs/client/HdfsDataInputStream.java | 38 +-
.../hdfs/client/HdfsDataOutputStream.java | 36 +-
.../org/apache/hadoop/hdfs/inotify/Event.java | 452 ++++++
.../apache/hadoop/hdfs/inotify/EventsList.java | 63 +
.../hdfs/inotify/MissingEventsException.java | 54 +
.../org/apache/hadoop/hdfs/protocol/Block.java | 14 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 131 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 79 +
.../hdfs/protocol/EncryptionZoneIterator.java | 51 +
.../hdfs/protocol/EncryptionZoneWithId.java | 81 +
.../protocol/EncryptionZoneWithIdIterator.java | 53 +
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 53 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 10 +-
.../hadoop/hdfs/protocol/LocatedBlocks.java | 23 +-
.../protocol/SnapshottableDirectoryStatus.java | 2 +-
.../datatransfer/ReplaceDatanodeOnFailure.java | 163 +-
.../WhitelistBasedTrustedChannelResolver.java | 119 ++
.../datatransfer/sasl/DataTransferSaslUtil.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 116 +-
.../ClientNamenodeProtocolTranslatorPB.java | 135 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 14 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 414 +++++-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 78 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 3 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 1392 ++++--------------
.../hdfs/server/balancer/BalancingPolicy.java | 84 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 1029 +++++++++++++
.../hadoop/hdfs/server/balancer/ExitStatus.java | 44 +
.../hadoop/hdfs/server/balancer/KeyManager.java | 173 +++
.../hadoop/hdfs/server/balancer/Matcher.java | 51 +
.../hdfs/server/balancer/MovedBlocks.java | 124 ++
.../hdfs/server/balancer/NameNodeConnector.java | 215 +--
.../hdfs/server/blockmanagement/BlockInfo.java | 35 +-
.../BlockInfoUnderConstruction.java | 12 +-
.../server/blockmanagement/BlockManager.java | 194 ++-
.../hdfs/server/blockmanagement/BlocksMap.java | 13 +-
.../CacheReplicationMonitor.java | 55 +-
.../blockmanagement/CorruptReplicasMap.java | 19 +-
.../blockmanagement/DatanodeDescriptor.java | 48 +-
.../server/blockmanagement/DatanodeManager.java | 21 +-
.../blockmanagement/DatanodeStorageInfo.java | 30 +-
.../blockmanagement/HeartbeatManager.java | 8 +
.../PendingDataNodeMessages.java | 34 +-
.../hdfs/server/common/HdfsServerConstants.java | 5 +
.../hdfs/server/datanode/BPOfferService.java | 305 ++--
.../hdfs/server/datanode/BPServiceActor.java | 15 +-
.../server/datanode/BlockPoolSliceStorage.java | 56 +-
.../hdfs/server/datanode/BlockReceiver.java | 59 +-
.../hdfs/server/datanode/BlockSender.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 51 +-
.../server/datanode/DataNodeLayoutVersion.java | 5 +-
.../hdfs/server/datanode/DataStorage.java | 352 ++++-
.../hdfs/server/datanode/DataXceiver.java | 26 +-
.../hdfs/server/datanode/DataXceiverServer.java | 40 +-
.../hdfs/server/datanode/DatanodeUtil.java | 36 +
.../hdfs/server/datanode/ReplicaInfo.java | 62 +-
.../server/datanode/ShortCircuitRegistry.java | 29 +-
.../hdfs/server/datanode/StorageLocation.java | 2 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 9 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 98 +-
.../impl/FsDatasetAsyncDiskService.java | 87 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 162 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 56 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 41 +-
.../server/datanode/fsdataset/impl/LDir.java | 228 ---
.../namenode/EditLogBackupInputStream.java | 5 +
.../server/namenode/EditLogFileInputStream.java | 5 +
.../server/namenode/EditLogInputStream.java | 6 +
.../namenode/EncryptionFaultInjector.java | 39 +
.../server/namenode/EncryptionZoneManager.java | 313 ++++
.../hdfs/server/namenode/FSDirectory.java | 324 +++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 67 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSImageFormat.java | 11 +
.../hdfs/server/namenode/FSNamesystem.java | 769 ++++++++--
.../server/namenode/FileJournalManager.java | 47 +-
.../hadoop/hdfs/server/namenode/INode.java | 59 +-
.../hdfs/server/namenode/INodeDirectory.java | 5 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 12 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 3 +-
.../hdfs/server/namenode/INodeReference.java | 4 +-
.../hdfs/server/namenode/INodeSymlink.java | 3 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 146 ++
.../hadoop/hdfs/server/namenode/JournalSet.java | 38 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 8 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 165 ++-
.../namenode/RedundantEditLogInputStream.java | 5 +
.../namenode/RetryStartFileException.java | 21 +
.../hdfs/server/namenode/TransferFsImage.java | 5 +-
.../server/namenode/XAttrPermissionFilter.java | 30 +-
.../namenode/metrics/FSNamesystemMBean.java | 7 +
.../snapshot/DirectoryWithSnapshotFeature.java | 10 +-
.../web/resources/NamenodeWebHdfsMethods.java | 26 +-
.../server/protocol/BlocksWithLocations.java | 25 +-
.../server/protocol/DatanodeStorageReport.java | 42 +
.../hdfs/server/protocol/RegisterCommand.java | 3 +
.../hadoop/hdfs/shortcircuit/DfsClientShm.java | 21 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 10 +-
.../hdfs/shortcircuit/DomainSocketFactory.java | 4 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 10 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 301 ++++
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../offlineEditsViewer/XmlEditsVisitor.java | 4 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 9 +-
.../apache/hadoop/hdfs/util/EnumDoubles.java | 128 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 +
.../hdfs/web/resources/FsActionParam.java | 58 +
.../hadoop/hdfs/web/resources/GetOpParam.java | 4 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 4 +
.../src/main/native/libhdfs/common/htable.c | 271 ++++
.../src/main/native/libhdfs/common/htable.h | 161 ++
.../src/main/native/libhdfs/exception.c | 65 +-
.../src/main/native/libhdfs/exception.h | 10 +-
.../src/main/native/libhdfs/expect.c | 8 +-
.../src/main/native/libhdfs/expect.h | 12 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 416 +++---
.../src/main/native/libhdfs/jni_helper.c | 265 ++--
.../src/main/native/libhdfs/jni_helper.h | 2 -
.../src/main/native/libhdfs/native_mini_dfs.c | 6 +-
.../src/main/native/libhdfs/os/mutexes.h | 55 +
.../src/main/native/libhdfs/os/posix/mutexes.c | 43 +
.../src/main/native/libhdfs/os/posix/platform.h | 34 +
.../src/main/native/libhdfs/os/posix/thread.c | 52 +
.../libhdfs/os/posix/thread_local_storage.c | 80 +
.../src/main/native/libhdfs/os/thread.h | 54 +
.../native/libhdfs/os/thread_local_storage.h | 75 +
.../main/native/libhdfs/os/windows/inttypes.h | 28 +
.../main/native/libhdfs/os/windows/mutexes.c | 52 +
.../main/native/libhdfs/os/windows/platform.h | 86 ++
.../src/main/native/libhdfs/os/windows/thread.c | 66 +
.../libhdfs/os/windows/thread_local_storage.c | 164 +++
.../src/main/native/libhdfs/os/windows/unistd.h | 29 +
.../main/native/libhdfs/test/test_libhdfs_ops.c | 232 +--
.../native/libhdfs/test/test_libhdfs_read.c | 19 +-
.../native/libhdfs/test/test_libhdfs_write.c | 29 +-
.../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 +-
.../main/native/libhdfs/test_libhdfs_threaded.c | 39 +-
.../main/native/libhdfs/test_native_mini_dfs.c | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 53 +
.../src/main/proto/DatanodeProtocol.proto | 24 -
.../src/main/proto/datatransfer.proto | 1 +
.../hadoop-hdfs/src/main/proto/encryption.proto | 65 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 47 +-
.../hadoop-hdfs/src/main/proto/inotify.proto | 117 ++
.../hadoop-hdfs/src/main/proto/xattr.proto | 1 +
.../src/main/resources/hdfs-default.xml | 56 +-
.../src/main/webapps/hdfs/dfshealth.html | 5 +-
.../src/main/webapps/hdfs/dfshealth.js | 11 +-
.../src/main/webapps/hdfs/index.html | 13 +-
.../src/main/webapps/secondary/index.html | 11 -
.../src/site/apt/ExtendedAttributes.apt.vm | 4 +-
.../src/site/apt/HDFSCommands.apt.vm | 408 +++++
.../src/site/apt/HdfsNfsGateway.apt.vm | 73 +-
.../src/site/apt/HdfsUserGuide.apt.vm | 29 +-
.../hadoop-hdfs/src/site/apt/LibHdfs.apt.vm | 29 +-
.../src/site/apt/TransparentEncryption.apt.vm | 206 +++
.../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 44 +
.../src/site/xdoc/HdfsRollingUpgrade.xml | 2 +-
.../org/apache/hadoop/TestGenericRefresh.java | 29 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 44 +-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 169 +++
.../hadoop/cli/util/CLICommandCryptoAdmin.java | 21 +
.../hadoop/cli/util/CryptoAdminCmdExecutor.java | 37 +
.../org/apache/hadoop/fs/TestGlobPaths.java | 260 ++--
.../java/org/apache/hadoop/fs/TestXAttr.java | 11 +-
.../fs/viewfs/TestViewFileSystemWithXAttrs.java | 151 ++
.../hadoop/fs/viewfs/TestViewFsWithXAttrs.java | 148 ++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 100 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 243 ++-
.../hdfs/MiniDFSClusterWithNodeGroup.java | 38 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 11 +
.../hadoop/hdfs/TestDFSClientRetries.java | 11 +-
.../org/apache/hadoop/hdfs/TestDFSFinalize.java | 9 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 430 ++++++
.../apache/hadoop/hdfs/TestDFSPermission.java | 77 +
.../org/apache/hadoop/hdfs/TestDFSRename.java | 44 +
.../org/apache/hadoop/hdfs/TestDFSRollback.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 177 ++-
.../hdfs/TestDFSStorageStateRecovery.java | 16 +-
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 12 +-
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 154 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 75 +-
.../hadoop/hdfs/TestDatanodeBlockScanner.java | 15 +-
.../apache/hadoop/hdfs/TestDatanodeConfig.java | 2 +
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 48 +
.../apache/hadoop/hdfs/TestDatanodeReport.java | 90 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 1 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 756 ++++++++++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 61 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 40 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestLease.java | 9 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 45 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 350 +++++
.../org/apache/hadoop/hdfs/TestSafeMode.java | 25 +-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 106 ++
.../apache/hadoop/hdfs/UpgradeUtilities.java | 44 +-
.../hdfs/crypto/TestHdfsCryptoStreams.java | 91 ++
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 21 +-
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 6 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +-
.../client/TestQuorumJournalManager.java | 2 +-
.../security/token/block/TestBlockToken.java | 2 +
.../hdfs/server/balancer/TestBalancer.java | 567 ++++++-
.../balancer/TestBalancerWithHANameNodes.java | 8 +-
.../TestBalancerWithMultipleNameNodes.java | 8 +-
.../balancer/TestBalancerWithNodeGroup.java | 12 +-
.../blockmanagement/BlockManagerTestUtil.java | 10 +
.../server/blockmanagement/TestBlockInfo.java | 20 +-
.../blockmanagement/TestBlockManager.java | 10 +-
.../blockmanagement/TestCorruptReplicaInfo.java | 14 +-
.../blockmanagement/TestDatanodeDescriptor.java | 6 +-
.../TestPendingDataNodeMessages.java | 7 +-
.../blockmanagement/TestReplicationPolicy.java | 5 +-
.../server/datanode/SimulatedFSDataset.java | 11 +
.../TestBlockHasMultipleReplicasOnSameDN.java | 5 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 1 -
.../server/datanode/TestBlockReplacement.java | 6 +-
.../datanode/TestDataNodeRollingUpgrade.java | 48 +
.../datanode/TestDataNodeVolumeFailure.java | 24 +-
.../hdfs/server/datanode/TestDataStorage.java | 210 +++
.../server/datanode/TestDeleteBlockPool.java | 7 +-
.../hdfs/server/datanode/TestDiskError.java | 4 +-
...TestDnRespectsBlockReportSplitThreshold.java | 6 +-
.../server/datanode/TestFsDatasetCache.java | 1 -
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 183 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 28 +
.../hdfs/server/namenode/FSXAttrBaseTest.java | 179 ++-
.../server/namenode/NNThroughputBenchmark.java | 5 +-
.../hdfs/server/namenode/TestAddBlockRetry.java | 4 +-
.../namenode/TestBlockUnderConstruction.java | 3 +
.../server/namenode/TestCacheDirectives.java | 1 -
.../TestCommitBlockSynchronization.java | 11 +
.../namenode/TestDecommissioningStatus.java | 98 +-
.../hdfs/server/namenode/TestDeleteRace.java | 133 ++
.../hdfs/server/namenode/TestEditLog.java | 4 +
.../hdfs/server/namenode/TestFSDirectory.java | 15 +-
.../server/namenode/TestFSNamesystemMBean.java | 2 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 19 +-
.../hdfs/server/namenode/TestINodeFile.java | 16 +
.../namenode/TestListCorruptFileBlocks.java | 163 +-
.../namenode/TestNNStorageRetentionManager.java | 11 +-
.../server/namenode/TestNameNodeMXBean.java | 45 +-
.../server/namenode/TestNamenodeRetryCache.java | 9 +-
.../hdfs/server/namenode/TestStartup.java | 41 +
.../namenode/ha/TestPipelinesFailover.java | 62 +-
.../namenode/ha/TestRetryCacheWithHA.java | 10 +-
.../namenode/metrics/TestNameNodeMetrics.java | 2 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 9 +
.../namenode/snapshot/TestSnapshotDeletion.java | 77 +-
.../shortcircuit/TestShortCircuitCache.java | 98 ++
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 39 +-
.../hadoop/hdfs/web/TestWebHDFSForHA.java | 76 +-
.../hdfs/web/TestWebHdfsFileSystemContract.java | 33 +
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 24 +
.../hadoop/security/TestPermissionSymlinks.java | 35 +
.../security/TestRefreshUserMappings.java | 12 +-
.../org/apache/hadoop/tracing/TestTracing.java | 280 ++++
.../src/test/resources/hadoop-0.23-reserved.tgz | Bin 0 -> 4558 bytes
.../src/test/resources/hadoop-1-reserved.tgz | Bin 0 -> 2572 bytes
.../test/resources/hadoop-24-datanode-dir.tgz | Bin 0 -> 637608 bytes
.../src/test/resources/hadoop-datanode-dir.txt | 23 +
.../src/test/resources/testCryptoConf.xml | 284 ++++
.../src/test/resources/testHDFSConf.xml | 44 +
.../src/test/resources/testXAttrConf.xml | 58 +-
hadoop-mapreduce-project/CHANGES.txt | 109 +-
hadoop-mapreduce-project/bin/mapred | 226 +--
hadoop-mapreduce-project/bin/mapred-config.sh | 76 +-
.../bin/mr-jobhistory-daemon.sh | 143 +-
hadoop-mapreduce-project/conf/mapred-env.sh | 60 +-
.../hadoop-mapreduce-client-app/pom.xml | 6 +
.../hadoop/mapred/LocalContainerLauncher.java | 18 +-
.../hadoop/mapred/TaskAttemptListenerImpl.java | 4 +-
.../org/apache/hadoop/mapred/YarnChild.java | 22 +-
.../jobhistory/JobHistoryEventHandler.java | 12 +-
.../hadoop/mapreduce/v2/app/AppContext.java | 1 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 5 +
.../v2/app/client/MRClientService.java | 3 +-
.../v2/app/commit/CommitterEventHandler.java | 2 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 30 +-
.../mapreduce/v2/app/rm/RMCommunicator.java | 2 +-
.../v2/app/rm/RMContainerAllocator.java | 26 +-
.../v2/app/speculate/DefaultSpeculator.java | 8 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 3 +-
.../mapreduce/v2/app/webapp/TasksBlock.java | 4 +-
.../mapred/TestTaskAttemptListenerImpl.java | 13 +-
.../jobhistory/TestJobHistoryEventHandler.java | 127 +-
.../hadoop/mapreduce/v2/app/MockAppContext.java | 5 +
.../mapreduce/v2/app/TestJobEndNotifier.java | 3 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 5 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 10 +-
.../mapreduce/v2/app/webapp/TestBlocks.java | 3 +
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +-
.../v2/jobhistory/JobHistoryUtils.java | 79 +-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 3 +-
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 9 +-
.../hadoop/mapreduce/v2/util/TestMRApps.java | 5 +-
.../org/apache/hadoop/mapred/BackupStore.java | 8 +-
.../java/org/apache/hadoop/mapred/IFile.java | 21 +-
.../org/apache/hadoop/mapred/InputFormat.java | 2 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 94 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 70 +-
.../java/org/apache/hadoop/mapred/Merger.java | 15 +-
.../java/org/apache/hadoop/mapred/Task.java | 6 +
.../hadoop/mapred/TaskCompletionEvent.java | 4 +-
.../mapred/join/CompositeInputFormat.java | 2 +-
.../apache/hadoop/mapreduce/CryptoUtils.java | 199 +++
.../apache/hadoop/mapreduce/InputFormat.java | 2 +-
.../java/org/apache/hadoop/mapreduce/Job.java | 11 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 9 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 23 +
.../hadoop/mapreduce/TaskCompletionEvent.java | 4 +-
.../mapreduce/lib/db/OracleDBRecordReader.java | 7 +-
.../lib/input/KeyValueTextInputFormat.java | 3 +
.../lib/join/CompositeInputFormat.java | 2 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 14 +-
.../mapreduce/task/reduce/LocalFetcher.java | 4 +
.../mapreduce/task/reduce/MergeManagerImpl.java | 37 +-
.../mapreduce/task/reduce/OnDiskMapOutput.java | 3 +-
.../task/reduce/ShuffleSchedulerImpl.java | 34 +-
.../hadoop/mapreduce/util/ConfigUtil.java | 2 +
.../src/main/resources/mapred-default.xml | 107 +-
.../src/site/apt/MapredCommands.apt.vm | 230 +++
.../apt/PluggableShuffleAndPluggableSort.apt.vm | 8 +-
.../src/site/markdown/DistCp.md.vm | 28 +-
.../src/site/markdown/HadoopArchives.md.vm | 27 +-
.../org/apache/hadoop/mapred/TestJobConf.java | 15 +-
.../hadoop/mapreduce/lib/db/TestDbClasses.java | 2 +-
.../mapreduce/task/reduce/TestMerger.java | 146 +-
.../mapreduce/v2/hs/HistoryClientService.java | 11 +-
...istoryServerFileSystemStateStoreService.java | 101 +-
.../hadoop/mapreduce/v2/hs/JobHistory.java | 5 +
.../mapreduce/v2/hs/server/HSAdminServer.java | 5 +-
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 70 +-
...istoryServerFileSystemStateStoreService.java | 85 +-
.../mapreduce/v2/hs/webapp/TestBlocks.java | 4 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 9 +
.../test/java/org/apache/hadoop/SleepJob.java | 275 ----
.../org/apache/hadoop/conf/TestJobConf.java | 20 +-
.../org/apache/hadoop/fs/JHLogAnalyzer.java | 4 +-
.../apache/hadoop/mapred/TestComparators.java | 61 +-
.../org/apache/hadoop/mapred/TestIFile.java | 11 +-
.../TestMRIntermediateDataEncryption.java | 254 ++++
.../apache/hadoop/mapred/TestMapOutputType.java | 62 +-
.../org/apache/hadoop/mapred/TestMapRed.java | 28 +-
.../apache/hadoop/mapred/TestReduceTask.java | 2 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 104 +-
.../mapred/lib/TestKeyFieldBasedComparator.java | 25 +-
.../mapred/pipes/TestPipeApplication.java | 12 +-
.../org/apache/hadoop/mapreduce/SleepJob.java | 32 +-
.../apache/hadoop/mapreduce/TestMapReduce.java | 30 +-
.../TestMRAMWithNonNormalizedCapabilities.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 +-
.../v2/TestMRJobsWithHistoryService.java | 2 +-
.../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 28 +-
.../hadoop/mapred/TestShuffleHandler.java | 8 +-
hadoop-project-dist/pom.xml | 14 +
hadoop-project/pom.xml | 49 +-
hadoop-project/src/site/site.xml | 5 +
.../org/apache/hadoop/tools/HadoopArchives.java | 14 +-
.../apache/hadoop/tools/TestHadoopArchives.java | 41 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 19 +
hadoop-tools/hadoop-aws/pom.xml | 116 ++
.../apache/hadoop/tools/DistCpConstants.java | 6 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 6 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 21 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 48 +-
.../hadoop/tools/mapred/CopyCommitter.java | 9 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 10 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 48 +-
.../hadoop/tools/TestDistCpWithRawXAttrs.java | 170 +++
.../hadoop/tools/TestDistCpWithXAttrs.java | 71 +-
.../apache/hadoop/tools/TestOptionsParser.java | 3 +-
.../hadoop/tools/util/DistCpTestUtils.java | 89 ++
.../hadoop/tools/util/TestDistCpUtils.java | 6 +-
.../mapred/gridmix/TestGridmixRecord.java | 10 +-
.../hadoop/mapred/gridmix/TestHighRamJob.java | 12 +-
hadoop-tools/hadoop-openstack/pom.xml | 5 +
.../hadoop/yarn/sls/RumenToSLSConverter.java | 4 +
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 4 +
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 73 +-
.../yarn/sls/appmaster/MRAMSimulator.java | 9 +-
.../hadoop/yarn/sls/conf/SLSConfiguration.java | 5 +
.../yarn/sls/nodemanager/NMSimulator.java | 72 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 35 +-
.../sls/scheduler/CapacitySchedulerMetrics.java | 5 +
.../yarn/sls/scheduler/ContainerSimulator.java | 4 +
.../sls/scheduler/FairSchedulerMetrics.java | 12 +-
.../sls/scheduler/FifoSchedulerMetrics.java | 4 +
.../NodeUpdateSchedulerEventWrapper.java | 4 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 15 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 50 +-
.../sls/scheduler/SLSCapacityScheduler.java | 26 +-
.../yarn/sls/scheduler/SchedulerMetrics.java | 4 +
.../yarn/sls/scheduler/SchedulerWrapper.java | 4 +
.../hadoop/yarn/sls/scheduler/TaskRunner.java | 22 +-
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 4 +
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 4 +
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 30 +-
.../yarn/sls/appmaster/TestAMSimulator.java | 86 ++
.../yarn/sls/nodemanager/TestNMSimulator.java | 100 ++
hadoop-tools/pom.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 194 ++-
hadoop-yarn-project/hadoop-yarn/bin/slaves.sh | 70 -
.../hadoop-yarn/bin/start-yarn.sh | 34 +-
.../hadoop-yarn/bin/stop-yarn.sh | 40 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 398 ++---
.../hadoop-yarn/bin/yarn-config.sh | 120 +-
.../hadoop-yarn/bin/yarn-daemon.sh | 157 +-
.../hadoop-yarn/bin/yarn-daemons.sh | 38 +-
.../hadoop-yarn/conf/yarn-env.sh | 163 +-
.../dev-support/findbugs-exclude.xml | 14 +
.../api/protocolrecords/AllocateResponse.java | 32 +
.../protocolrecords/GetApplicationsRequest.java | 18 +
.../hadoop/yarn/api/records/ContainerId.java | 10 +-
.../hadoop/yarn/api/records/ResourceOption.java | 4 +
.../hadoop/yarn/conf/YarnConfiguration.java | 40 +-
.../UpdateNodeResourceResponse.java | 12 +-
..._server_resourcemanager_service_protos.proto | 63 -
.../src/main/proto/yarn_service_protos.proto | 1 +
.../distributedshell/ApplicationMaster.java | 31 +-
.../applications/distributedshell/Client.java | 2 +-
.../distributedshell/TestDSFailedAppMaster.java | 6 +-
.../distributedshell/TestDistributedShell.java | 75 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 8 -
.../hadoop/yarn/client/api/AMRMClient.java | 63 +
.../hadoop/yarn/client/api/TimelineClient.java | 88 --
.../yarn/client/api/async/AMRMClientAsync.java | 64 +
.../yarn/client/api/impl/AMRMClientImpl.java | 19 +
.../client/api/impl/TimelineAuthenticator.java | 260 ----
.../client/api/impl/TimelineClientImpl.java | 332 -----
.../hadoop/yarn/client/ProtocolHATestBase.java | 1 +
.../TestApplicationMasterServiceOnHA.java | 6 +-
.../api/async/impl/TestAMRMClientAsync.java | 76 +-
.../yarn/client/api/impl/TestAMRMClient.java | 170 ++-
.../api/impl/TestAMRMClientOnRMRestart.java | 151 ++
.../api/impl/TestTimelineAuthenticator.java | 40 -
.../client/api/impl/TestTimelineClient.java | 206 ---
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 +
.../impl/pb/AllocateResponsePBImpl.java | 38 +-
.../impl/pb/GetApplicationsRequestPBImpl.java | 10 +
.../impl/pb/ApplicationReportPBImpl.java | 1 +
.../pb/ApplicationSubmissionContextPBImpl.java | 1 +
.../impl/pb/ResourceBlacklistRequestPBImpl.java | 13 +-
.../records/impl/pb/ResourceOptionPBImpl.java | 48 +-
.../yarn/api/records/impl/pb/TokenPBImpl.java | 2 +-
.../hadoop/yarn/client/api/TimelineClient.java | 88 ++
.../client/api/impl/TimelineAuthenticator.java | 269 ++++
.../client/api/impl/TimelineClientImpl.java | 406 +++++
.../yarn/client/api/impl/package-info.java | 21 +
.../hadoop/yarn/client/api/package-info.java | 21 +
.../pb/UpdateNodeResourceRequestPBImpl.java | 16 +-
.../pb/UpdateNodeResourceResponsePBImpl.java | 2 +-
.../yarn/util/ApplicationClassLoader.java | 170 +--
.../resource/DefaultResourceCalculator.java | 7 +
.../resource/DominantResourceCalculator.java | 8 +
.../yarn/util/resource/ResourceCalculator.java | 9 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../hadoop/yarn/webapp/util/WebAppUtils.java | 83 +-
.../src/main/resources/yarn-default.xml | 68 +-
.../apache/hadoop/yarn/api/TestContainerId.java | 4 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 895 +++++++++++
.../api/impl/TestTimelineAuthenticator.java | 40 +
.../client/api/impl/TestTimelineClient.java | 206 +++
.../hadoop/yarn/conf/TestYarnConfiguration.java | 128 ++
.../yarn/util/TestApplicationClassLoader.java | 136 --
.../yarn/webapp/util/TestWebAppUtils.java | 148 ++
.../ApplicationHistoryClientService.java | 16 +-
.../ApplicationHistoryServer.java | 117 +-
.../FileSystemApplicationHistoryStore.java | 14 +-
.../webapp/AHSWebApp.java | 25 +-
.../server/timeline/LeveldbTimelineStore.java | 74 +-
.../server/timeline/TimelineDataManager.java | 319 ++++
.../timeline/webapp/CrossOriginFilter.java | 220 +++
.../webapp/CrossOriginFilterInitializer.java | 42 +
.../timeline/webapp/TimelineWebServices.java | 255 +---
.../TestApplicationHistoryClientService.java | 2 +-
.../TestApplicationHistoryServer.java | 41 +
.../TestFileSystemApplicationHistoryStore.java | 62 +-
.../timeline/TestLeveldbTimelineStore.java | 53 +-
.../timeline/webapp/TestCrossOriginFilter.java | 214 +++
.../TestCrossOriginFilterInitializer.java | 60 +
.../webapp/TestTimelineWebServices.java | 9 +-
.../webapp/TestTimelineWebServicesWithSSL.java | 134 ++
.../hadoop/yarn/server/records/Version.java | 84 ++
.../server/records/impl/pb/VersionPBImpl.java | 77 +
.../security/http/RMAuthenticationFilter.java | 63 +
.../http/RMAuthenticationFilterInitializer.java | 121 ++
.../main/proto/yarn_server_common_protos.proto | 8 +-
.../server/nodemanager/ContainerExecutor.java | 73 +-
.../nodemanager/DefaultContainerExecutor.java | 59 +-
.../nodemanager/LinuxContainerExecutor.java | 25 +-
.../server/nodemanager/NodeStatusUpdater.java | 23 +
.../nodemanager/NodeStatusUpdaterImpl.java | 34 +-
.../containermanager/ContainerManagerImpl.java | 260 +++-
.../application/ApplicationImpl.java | 6 +
.../container/ContainerImpl.java | 165 ++-
.../launcher/ContainerLaunch.java | 46 +-
.../launcher/ContainersLauncher.java | 15 +-
.../launcher/ContainersLauncherEventType.java | 1 +
.../launcher/RecoveredContainerLaunch.java | 126 ++
.../localizer/ResourceLocalizationService.java | 10 +-
.../logaggregation/AppLogAggregator.java | 2 +
.../logaggregation/AppLogAggregatorImpl.java | 14 +-
.../logaggregation/LogAggregationService.java | 10 +-
.../recovery/NMLeveldbStateStoreService.java | 287 +++-
.../recovery/NMNullStateStoreService.java | 58 +
.../recovery/NMStateStoreService.java | 212 +++
.../recovery/records/NMDBSchemaVersion.java | 80 -
.../impl/pb/NMDBSchemaVersionPBImpl.java | 81 -
.../server/nodemanager/webapp/WebServer.java | 4 +-
.../impl/container-executor.c | 111 +-
.../yarn_server_nodemanager_recovery.proto | 12 +-
.../nodemanager/TestLinuxContainerExecutor.java | 7 +
.../nodemanager/TestNodeManagerResync.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 17 +-
.../BaseContainerManagerTest.java | 2 +-
.../containermanager/TestAuxServices.java | 3 +-
.../TestContainerManagerRecovery.java | 324 ++++
.../container/TestContainer.java | 6 +-
.../recovery/NMMemoryStateStoreService.java | 152 +-
.../TestNMLeveldbStateStoreService.java | 187 ++-
.../nodemanager/webapp/TestNMWebServer.java | 2 +-
.../conf/capacity-scheduler.xml | 23 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 31 +
.../server/resourcemanager/AdminService.java | 50 +-
.../ApplicationMasterService.java | 41 +-
.../server/resourcemanager/ClientRMService.java | 8 +-
.../resourcemanager/EmbeddedElectorService.java | 10 +-
.../resourcemanager/RMSecretManagerService.java | 6 +-
.../server/resourcemanager/ResourceManager.java | 103 +-
.../resourcemanager/ResourceTrackerService.java | 14 +-
.../resourcemanager/amlauncher/AMLauncher.java | 11 +-
.../recovery/FileSystemRMStateStore.java | 87 +-
.../recovery/MemoryRMStateStore.java | 21 +-
.../recovery/NullRMStateStore.java | 13 +-
.../resourcemanager/recovery/RMStateStore.java | 108 +-
.../recovery/RMStateStoreFactory.java | 13 +-
.../recovery/ZKRMStateStore.java | 78 +-
.../records/AMRMTokenSecretManagerState.java | 76 +
.../records/ApplicationAttemptStateData.java | 2 +-
.../recovery/records/ApplicationStateData.java | 2 +-
.../resourcemanager/recovery/records/Epoch.java | 2 +-
.../recovery/records/RMStateVersion.java | 80 -
.../pb/AMRMTokenSecretManagerStatePBImpl.java | 126 ++
.../pb/ApplicationAttemptStateDataPBImpl.java | 6 +-
.../impl/pb/ApplicationStateDataPBImpl.java | 6 +-
.../recovery/records/impl/pb/EpochPBImpl.java | 4 +-
.../records/impl/pb/RMStateVersionPBImpl.java | 76 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 40 +-
.../rmapp/RMAppNewSavedEvent.java | 36 -
.../rmapp/RMAppUpdateSavedEvent.java | 36 -
.../rmapp/attempt/RMAppAttemptImpl.java | 84 +-
.../event/RMAppAttemptNewSavedEvent.java | 39 -
.../event/RMAppAttemptUpdateSavedEvent.java | 38 -
.../server/resourcemanager/rmnode/RMNode.java | 12 -
.../resourcemanager/rmnode/RMNodeEventType.java | 3 +
.../resourcemanager/rmnode/RMNodeImpl.java | 162 +-
.../rmnode/RMNodeReconnectEvent.java | 12 +-
.../rmnode/RMNodeResourceUpdateEvent.java | 37 +
.../scheduler/AbstractYarnScheduler.java | 185 +++
.../scheduler/AppSchedulingInfo.java | 5 +-
.../scheduler/SchedulerApplicationAttempt.java | 18 +-
.../scheduler/SchedulerNode.java | 25 +-
.../scheduler/SchedulerUtils.java | 32 -
.../scheduler/YarnScheduler.java | 18 +
.../scheduler/capacity/CSQueue.java | 18 +
.../scheduler/capacity/CSQueueUtils.java | 53 +
.../scheduler/capacity/CapacityScheduler.java | 223 ++-
.../CapacitySchedulerConfiguration.java | 119 +-
.../scheduler/capacity/LeafQueue.java | 50 +-
.../scheduler/capacity/ParentQueue.java | 33 +
.../event/NodeResourceUpdateSchedulerEvent.java | 43 +
.../scheduler/event/SchedulerEventType.java | 1 +
.../scheduler/fair/AllocationConfiguration.java | 40 +-
.../fair/AllocationFileLoaderService.java | 57 +-
.../scheduler/fair/AppSchedulable.java | 463 ------
.../scheduler/fair/FSAppAttempt.java | 762 ++++++++++
.../scheduler/fair/FSLeafQueue.java | 85 +-
.../scheduler/fair/FSOpDurations.java | 119 ++
.../scheduler/fair/FSParentQueue.java | 25 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 85 +-
.../scheduler/fair/FSQueueMetrics.java | 17 +-
.../scheduler/fair/FSSchedulerApp.java | 360 -----
.../scheduler/fair/FSSchedulerNode.java | 6 +-
.../scheduler/fair/FairScheduler.java | 227 +--
.../scheduler/fair/FifoAppComparator.java | 8 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 52 +-
.../scheduler/fair/NewAppWeightBooster.java | 2 +-
.../scheduler/fair/QueueManager.java | 37 +-
.../scheduler/fair/Schedulable.java | 61 +-
.../scheduler/fair/SchedulingPolicy.java | 27 +-
.../scheduler/fair/WeightAdjuster.java | 2 +-
.../fair/policies/ComputeFairShares.java | 55 +-
.../DominantResourceFairnessPolicy.java | 9 +
.../fair/policies/FairSharePolicy.java | 8 +
.../scheduler/fair/policies/FifoPolicy.java | 8 +
.../scheduler/fifo/FifoScheduler.java | 42 +-
.../security/AMRMTokenSecretManager.java | 53 +-
.../security/DelegationTokenRenewer.java | 2 +-
.../security/RMAuthenticationHandler.java | 157 ++
.../resourcemanager/webapp/AboutBlock.java | 1 +
.../webapp/FairSchedulerAppsBlock.java | 4 +
.../webapp/FairSchedulerPage.java | 31 +-
.../resourcemanager/webapp/RMWebServices.java | 26 +-
.../resourcemanager/webapp/dao/ClusterInfo.java | 8 +
.../webapp/dao/FairSchedulerInfo.java | 16 +-
.../webapp/dao/FairSchedulerLeafQueueInfo.java | 9 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 34 +-
.../yarn_server_resourcemanager_recovery.proto | 90 ++
.../yarn/server/resourcemanager/MockAM.java | 8 +-
.../yarn/server/resourcemanager/MockNodes.java | 22 +-
.../yarn/server/resourcemanager/MockRM.java | 15 +-
.../MockRMWithCustomAMLauncher.java | 5 +-
.../resourcemanager/TestApplicationCleanup.java | 29 +
.../TestApplicationMasterService.java | 1 -
.../resourcemanager/TestFifoScheduler.java | 85 ++
.../yarn/server/resourcemanager/TestRMHA.java | 46 +-
.../resourcemanager/TestRMNodeTransitions.java | 80 +-
.../server/resourcemanager/TestRMRestart.java | 25 +-
.../resourcemanager/TestResourceManager.java | 76 +
.../TestResourceTrackerService.java | 2 +-
.../TestWorkPreservingRMRestart.java | 101 +-
.../applicationsmanager/TestAMRestart.java | 9 +-
.../recovery/RMStateStoreTestBase.java | 108 +-
.../recovery/TestFSRMStateStore.java | 14 +-
.../recovery/TestZKRMStateStore.java | 13 +-
.../resourcetracker/TestNMReconnect.java | 19 +-
.../rmapp/TestRMAppTransitions.java | 15 +-
.../attempt/TestRMAppAttemptTransitions.java | 48 +-
.../scheduler/capacity/TestCSQueueUtils.java | 262 ++++
.../capacity/TestCapacityScheduler.java | 983 ++++++++++++-
.../scheduler/capacity/TestQueueMappings.java | 240 +++
.../scheduler/capacity/TestUtils.java | 5 +-
.../scheduler/fair/FakeSchedulable.java | 13 +-
.../fair/TestAllocationFileLoaderService.java | 64 +-
.../scheduler/fair/TestFSAppAttempt.java | 188 +++
.../scheduler/fair/TestFSLeafQueue.java | 2 +-
.../scheduler/fair/TestFSSchedulerApp.java | 191 ---
.../scheduler/fair/TestFairScheduler.java | 684 ++++++++-
.../fair/TestFairSchedulerFairShare.java | 360 +++++
.../fair/TestMaxRunningAppsEnforcer.java | 42 +-
.../scheduler/fifo/TestFifoScheduler.java | 24 +-
.../security/TestAMRMTokens.java | 55 +-
.../webapp/TestRMWebAppFairScheduler.java | 96 +-
.../webapp/TestRMWebServices.java | 2 +-
.../TestRMWebServicesAppsModification.java | 21 +-
...ebServicesDelegationTokenAuthentication.java | 357 +++++
.../webapp/TestRMWebappAuthentication.java | 272 ++++
.../src/site/apt/FairScheduler.apt.vm | 38 +-
.../src/site/apt/ResourceManagerRest.apt.vm | 21 +
.../src/site/apt/WritingYarnApplications.apt.vm | 1369 +++++++++--------
977 files changed, 60546 insertions(+), 16363 deletions(-)
----------------------------------------------------------------------
[16/50] [abbrv] git commit: MAPREDUCE-5984. native-task: Reuse lz4
sources in hadoop-common. Contributed by Binglin Chang
Posted by ar...@apache.org.
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common. Contributed by Binglin Chang
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616105 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ecaa81d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ecaa81d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ecaa81d
Branch: refs/heads/HDFS-6581
Commit: 7ecaa81d2746a805b9937fcd134ceaa8607ed7b5
Parents: 83a3967
Author: Binglin Chang <bc...@apache.org>
Authored: Wed Aug 6 06:01:12 2014 +0000
Committer: Binglin Chang <bc...@apache.org>
Committed: Wed Aug 6 06:01:12 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 6 +
.../src/CMakeLists.txt | 4 +-
.../src/main/native/COPYING | 12 +-
.../src/main/native/lz4/lz4.c | 740 -------------------
.../src/main/native/lz4/lz4.h | 96 ---
.../src/main/native/src/codec/Lz4Codec.cc | 22 +-
.../src/main/native/test/TestCompressions.cc | 6 +-
.../src/main/native/test/TestIFile.cc | 2 +-
9 files changed, 21 insertions(+), 868 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index c70df12..9dddcd5 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -9,3 +9,4 @@ MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
+MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 2cb483e..9727800 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -163,6 +163,12 @@
</goals>
<configuration>
<target>
+ <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h"
+ todir="${project.build.directory}/native/" />
+ <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h"
+ todir="${project.build.directory}/native/" />
+ <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c"
+ todir="${project.build.directory}/native/" />
<copy todir="${project.build.directory}/native/test/testData"
overwrite="true">
<fileset dir="${basedir}/src/main/native/testData" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 3094162..36dbd9c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -155,7 +155,7 @@ include_directories(
${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
#${CMAKE_CURRENT_SOURCE_DIR}/src
- #${CMAKE_BINARY_DIR}
+ ${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
@@ -174,7 +174,7 @@ else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
add_dual_library(nativetask
- ${D}/lz4/lz4.c
+ ${CMAKE_BINARY_DIR}/lz4.c
${D}/cityhash/city.cc
${D}/src/codec/BlockCodec.cc
${D}/src/codec/GzipCodec.cc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
index c5d3420..1d60d1d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
@@ -56,20 +56,20 @@ LZ4
---------------------------------------------------------------------
LZ4 - Fast LZ compression algorithm
Header File
- Copyright (C) 2011, Yann Collet.
- BSD License
+ Copyright (C) 2011-2014, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
-
+
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
-
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -81,3 +81,7 @@ LZ4
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.c
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.c b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.c
deleted file mode 100644
index 345a436..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.c
+++ /dev/null
@@ -1,740 +0,0 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2012, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-//**************************************
-// Compilation Directives
-//**************************************
-#if __STDC_VERSION__ >= 199901L
- /* "restrict" is a known keyword */
-#else
-#define restrict // Disable restrict
-#endif
-
-#ifdef _MSC_VER
-#define inline __forceinline
-#endif
-
-#ifdef __GNUC__
-#define _PACKED __attribute__ ((packed))
-#else
-#define _PACKED
-#endif
-
-#if (__x86_64__ || __ppc64__ || _WIN64 || __LP64__) // Detect 64 bits mode
-#define ARCH64 1
-#else
-#define ARCH64 0
-#endif
-
-
-//**************************************
-// Includes
-//**************************************
-#include <stdlib.h> // for malloc
-#include <string.h> // for memset
-#include "lz4.h"
-
-
-//**************************************
-// Performance parameter
-//**************************************
-// Increasing this value improves compression ratio
-// Lowering this value reduces memory usage
-// Lowering may also improve speed, typically on reaching cache size limits (L1 32KB for Intel, 64KB for AMD)
-// Memory usage formula for 32 bits systems : N->2^(N+2) Bytes (examples : 17 -> 512KB ; 12 -> 16KB)
-#define HASH_LOG 12
-
-//#define _FORCE_SW_BITCOUNT // Uncomment for better performance if target platform has no hardware support for LowBitCount
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively
-#define BYTE unsigned __int8
-#define U16 unsigned __int16
-#define U32 unsigned __int32
-#define S32 __int32
-#define U64 unsigned __int64
-#else
-#include <stdint.h>
-#define BYTE uint8_t
-#define U16 uint16_t
-#define U32 uint32_t
-#define S32 int32_t
-#define U64 uint64_t
-#endif
-
-
-//**************************************
-// Constants
-//**************************************
-#define MINMATCH 4
-#define SKIPSTRENGTH 6
-#define STACKLIMIT 13
-#define HEAPMODE (HASH_LOG>STACKLIMIT) // Defines if memory is allocated into the stack (local variable), or into the heap (malloc()).
-#define COPYLENGTH 8
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH+MINMATCH)
-#define MINLENGTH (MFLIMIT+1)
-
-#define MAXD_LOG 16
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
-
-#define HASHTABLESIZE (1 << HASH_LOG)
-#define HASH_MASK (HASHTABLESIZE - 1)
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-
-//**************************************
-// Local structures
-//**************************************
-struct refTables
-{
- const BYTE* hashTable[HASHTABLESIZE];
-};
-
-typedef struct _U64_S
-{
- U64 v;
-} _PACKED U64_S;
-
-typedef struct _U32_S
-{
- U32 v;
-} _PACKED U32_S;
-
-typedef struct _U16_S
-{
- U16 v;
-} _PACKED U16_S;
-
-#define A64(x) (((U64_S *)(x))->v)
-#define A32(x) (((U32_S *)(x))->v)
-#define A16(x) (((U16_S *)(x))->v)
-
-
-//**************************************
-// Architecture-specific macros
-//**************************************
-#if ARCH64 // 64-bit
-#define STEPSIZE 8
-#define UARCH U64
-#define AARCH A64
-#define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
-#define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
-#define LZ4_SECURECOPY(s,d,e) if (d<e) LZ4_WILDCOPY(s,d,e)
-#define HTYPE U32
-#define INITBASE(base) const BYTE* const base = ip
-#else // 32-bit
-#define STEPSIZE 4
-#define UARCH U32
-#define AARCH A32
-#define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
-#define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
-#define LZ4_SECURECOPY LZ4_WILDCOPY
-#define HTYPE const BYTE*
-#define INITBASE(base) const int base = 0
-#endif
-
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
-#define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
-#define LZ4_NbCommonBytes LZ4_NbCommonBytes_LittleEndian
-#else // Big Endian
-#define LZ4_READ_LITTLEENDIAN_16(d,s,p) { int delta = p[0]; delta += p[1] << 8; d = (s) - delta; }
-#define LZ4_WRITE_LITTLEENDIAN_16(p,v) { int delta = v; *p++ = delta; *p++ = delta>>8; }
-#define LZ4_NbCommonBytes LZ4_NbCommonBytes_BigEndian
-#endif
-
-
-//**************************************
-// Macros
-//**************************************
-#define LZ4_HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
-#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p))
-#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
-#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=(d)+l; LZ4_WILDCOPY(s,d,e); d=e; }
-
-
-//****************************
-// Private functions
-//****************************
-#if ARCH64
-
-inline static int LZ4_NbCommonBytes_LittleEndian (register U64 val)
-{
- #if defined(_MSC_VER) && !defined(_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanForward64( &r, val );
- return (int)(r>>3);
- #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll(val) >> 3);
- #else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58];
- #endif
-}
-
-inline static int LZ4_NbCommonBytes_BigEndian (register U64 val)
-{
- #if defined(_MSC_VER) && !defined(_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (int)(r>>3);
- #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
- #else
- int r;
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
- #endif
-}
-
-#else
-
-inline static int LZ4_NbCommonBytes_LittleEndian (register U32 val)
-{
- #if defined(_MSC_VER) && !defined(_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanForward( &r, val );
- return (int)(r>>3);
- #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT)
- return (__builtin_ctz(val) >> 3);
- #else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -val) * 0x077CB531U)) >> 27];
- #endif
-}
-
-inline static int LZ4_NbCommonBytes_BigEndian (register U32 val)
-{
- #if defined(_MSC_VER) && !defined(_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse( &r, val );
- return (int)(r>>3);
- #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT)
- return (__builtin_clz(val) >> 3);
- #else
- int r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
- #endif
-}
-
-#endif
-
-
-//******************************
-// Public Compression functions
-//******************************
-
-int LZ4_compressCtx(void** ctx,
- char* source,
- char* dest,
- int isize)
-{
-#if HEAPMODE
- struct refTables *srt = (struct refTables *) (*ctx);
- HTYPE* HashTable;
-#else
- HTYPE HashTable[HASHTABLESIZE] = {0};
-#endif
-
- const BYTE* ip = (BYTE*) source;
- INITBASE(base);
- const BYTE* anchor = ip;
- const BYTE* const iend = ip + isize;
- const BYTE* const mflimit = iend - MFLIMIT;
-#define matchlimit (iend - LASTLITERALS)
-
- BYTE* op = (BYTE*) dest;
-
- int len, length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
-
- // Init
- if (isize<MINLENGTH) goto _last_literals;
-#if HEAPMODE
- if (*ctx == NULL)
- {
- srt = (struct refTables *) malloc ( sizeof(struct refTables) );
- *ctx = (void*) srt;
- }
- HashTable = (HTYPE*)(srt->hashTable);
- memset((void*)HashTable, 0, sizeof(srt->hashTable));
-#else
- (void) ctx;
-#endif
-
-
- // First Byte
- HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
- ip++; forwardH = LZ4_HASH_VALUE(ip);
-
- // Main Loop
- for ( ; ; )
- {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE* forwardIp = ip;
- const BYTE* ref;
- BYTE* token;
-
- // Find a match
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if (forwardIp > mflimit) { goto _last_literals; }
-
- forwardH = LZ4_HASH_VALUE(forwardIp);
- ref = base + HashTable[h];
- HashTable[h] = ip - base;
-
- } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
-
- // Catch up
- while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; }
-
- // Encode Literal length
- length = ip - anchor;
- token = op++;
- if (length>=(int)RUN_MASK) { *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; }
- else *token = (length<<ML_BITS);
-
- // Copy Literals
- LZ4_BLINDCOPY(anchor, op, length);
-
-_next_match:
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(op,ip-ref);
-
- // Start Counting
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified
- anchor = ip;
- while (ip<matchlimit-(STEPSIZE-1))
- {
- UARCH diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
- if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
- if ((ip<matchlimit) && (*ref == *ip)) ip++;
-_endCount:
-
- // Encode MatchLength
- len = (ip - anchor);
- if (len>=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; }
- else *token += len;
-
- // Test end of chunk
- if (ip > mflimit) { anchor = ip; break; }
-
- // Fill table
- HashTable[LZ4_HASH_VALUE(ip-2)] = ip - 2 - base;
-
- // Test next position
- ref = base + HashTable[LZ4_HASH_VALUE(ip)];
- HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
- if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
-
- // Prepare next loop
- anchor = ip++;
- forwardH = LZ4_HASH_VALUE(ip);
- }
-
-_last_literals:
- // Encode Last Literals
- {
- int lastRun = iend - anchor;
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-
-// Note : this function is valid only if isize < LZ4_64KLIMIT
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))
-#define HASHLOG64K (HASH_LOG+1)
-#define HASH64KTABLESIZE (1U<<HASHLOG64K)
-#define LZ4_HASH64K_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASHLOG64K))
-#define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p))
-int LZ4_compress64kCtx(void** ctx,
- char* source,
- char* dest,
- int isize)
-{
-#if HEAPMODE
- struct refTables *srt = (struct refTables *) (*ctx);
- U16* HashTable;
-#else
- U16 HashTable[HASH64KTABLESIZE] = {0};
-#endif
-
- const BYTE* ip = (BYTE*) source;
- const BYTE* anchor = ip;
- const BYTE* const base = ip;
- const BYTE* const iend = ip + isize;
- const BYTE* const mflimit = iend - MFLIMIT;
-#define matchlimit (iend - LASTLITERALS)
-
- BYTE* op = (BYTE*) dest;
-
- int len, length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
-
- // Init
- if (isize<MINLENGTH) goto _last_literals;
-#if HEAPMODE
- if (*ctx == NULL)
- {
- srt = (struct refTables *) malloc ( sizeof(struct refTables) );
- *ctx = (void*) srt;
- }
- HashTable = (U16*)(srt->hashTable);
- memset((void*)HashTable, 0, sizeof(srt->hashTable));
-#else
- (void) ctx;
-#endif
-
-
- // First Byte
- ip++; forwardH = LZ4_HASH64K_VALUE(ip);
-
- // Main Loop
- for ( ; ; )
- {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE* forwardIp = ip;
- const BYTE* ref;
- BYTE* token;
-
- // Find a match
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if (forwardIp > mflimit) { goto _last_literals; }
-
- forwardH = LZ4_HASH64K_VALUE(forwardIp);
- ref = base + HashTable[h];
- HashTable[h] = ip - base;
-
- } while (A32(ref) != A32(ip));
-
- // Catch up
- while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; }
-
- // Encode Literal length
- length = ip - anchor;
- token = op++;
- if (length>=(int)RUN_MASK) { *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; }
- else *token = (length<<ML_BITS);
-
- // Copy Literals
- LZ4_BLINDCOPY(anchor, op, length);
-
-_next_match:
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(op,ip-ref);
-
- // Start Counting
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified
- anchor = ip;
- while (ip<matchlimit-(STEPSIZE-1))
- {
- UARCH diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
- if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
- if ((ip<matchlimit) && (*ref == *ip)) ip++;
-_endCount:
-
- // Encode MatchLength
- len = (ip - anchor);
- if (len>=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; }
- else *token += len;
-
- // Test end of chunk
- if (ip > mflimit) { anchor = ip; break; }
-
- // Fill table
- HashTable[LZ4_HASH64K_VALUE(ip-2)] = ip - 2 - base;
-
- // Test next position
- ref = base + HashTable[LZ4_HASH64K_VALUE(ip)];
- HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base;
- if (A32(ref) == A32(ip)) { token = op++; *token=0; goto _next_match; }
-
- // Prepare next loop
- anchor = ip++;
- forwardH = LZ4_HASH64K_VALUE(ip);
- }
-
-_last_literals:
- // Encode Last Literals
- {
- int lastRun = iend - anchor;
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-
-int LZ4_compress(char* source,
- char* dest,
- int isize)
-{
-#if HEAPMODE
- void* ctx = malloc(sizeof(struct refTables));
- int result;
- if (isize < LZ4_64KLIMIT)
- result = LZ4_compress64kCtx(&ctx, source, dest, isize);
- else result = LZ4_compressCtx(&ctx, source, dest, isize);
- free(ctx);
- return result;
-#else
- if (isize < (int)LZ4_64KLIMIT) return LZ4_compress64kCtx(NULL, source, dest, isize);
- return LZ4_compressCtx(NULL, source, dest, isize);
-#endif
-}
-
-
-
-
-//****************************
-// Decompression functions
-//****************************
-
-// Note : The decoding functions LZ4_uncompress() and LZ4_uncompress_unknownOutputSize()
-// are safe against "buffer overflow" attack type.
-// They will never write nor read outside of the provided input and output buffers.
-// A corrupted input will produce an error result, a negative int, indicating the position of the error within input stream.
-
-int LZ4_uncompress(char* source,
- char* dest,
- int osize)
-{
- // Local Variables
- const BYTE* restrict ip = (const BYTE*) source;
- const BYTE* restrict ref;
-
- BYTE* restrict op = (BYTE*) dest;
- BYTE* const oend = op + osize;
- BYTE* cpy;
-
- BYTE token;
-
- int len, length;
- size_t dec[] ={0, 3, 2, 3, 0, 0, 0, 0};
-
-
- // Main Loop
- while (1)
- {
- // get runlength
- token = *ip++;
- if ((length=(token>>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }
-
- // copy literals
- cpy = op+length;
- if (cpy>oend-COPYLENGTH)
- {
- if (cpy > oend) goto _output_error;
- memcpy(op, ip, length);
- ip += length;
- break; // Necessarily EOF
- }
- LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy;
-
- // get offset
- LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if (ref < (BYTE* const)dest) goto _output_error;
-
- // get matchlength
- if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; }
-
- // copy repeated sequence
- if (op-ref<STEPSIZE)
- {
-#if ARCH64
- size_t dec2table[]={0, 4, 4, 3, 4, 5, 6, 7};
- size_t dec2 = dec2table[op-ref];
-#else
- const int dec2 = 0;
-#endif
- *op++ = *ref++;
- *op++ = *ref++;
- *op++ = *ref++;
- *op++ = *ref++;
- ref -= dec[op-ref];
- A32(op)=A32(ref); op += STEPSIZE-4; ref += STEPSIZE-4;
- ref -= dec2;
- } else { LZ4_COPYSTEP(ref,op); }
- cpy = op + length - (STEPSIZE-4);
- if (cpy>oend-COPYLENGTH)
- {
- if (cpy > oend) goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH));
- while(op<cpy) *op++=*ref++;
- op=cpy;
- if (op == oend) break; // Check EOF (should never happen, since last 5 bytes are supposed to be literals)
- continue;
- }
- LZ4_SECURECOPY(ref, op, cpy);
- op=cpy; // correction
- }
-
- // end of decoding
- return (int) (((char*)ip)-source);
-
- // write overflow error detected
-_output_error:
- return (int) (-(((char*)ip)-source));
-}
-
-
-int LZ4_uncompress_unknownOutputSize(
- char* source,
- char* dest,
- int isize,
- int maxOutputSize)
-{
- // Local Variables
- const BYTE* restrict ip = (const BYTE*) source;
- const BYTE* const iend = ip + isize;
- const BYTE* restrict ref;
-
- BYTE* restrict op = (BYTE*) dest;
- BYTE* const oend = op + maxOutputSize;
- BYTE* cpy;
-
- BYTE token;
-
- int len, length;
- size_t dec[] ={0, 3, 2, 3, 0, 0, 0, 0};
-
-
- // Main Loop
- while (ip<iend)
- {
- // get runlength
- token = *ip++;
- if ((length=(token>>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }
-
- // copy literals
- cpy = op+length;
- if (cpy>oend-COPYLENGTH)
- {
- if (cpy > oend) goto _output_error;
- memcpy(op, ip, length);
- op += length;
- break; // Necessarily EOF
- }
- LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy;
- if (ip>=iend) break; // check EOF
-
- // get offset
- LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if (ref < (BYTE* const)dest) goto _output_error;
-
- // get matchlength
- if ((length=(token&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }
-
- // copy repeated sequence
- if (op-ref<STEPSIZE)
- {
-#if ARCH64
- size_t dec2table[]={0, 4, 4, 3, 4, 5, 6, 7};
- size_t dec2 = dec2table[op-ref];
-#else
- const int dec2 = 0;
-#endif
- *op++ = *ref++;
- *op++ = *ref++;
- *op++ = *ref++;
- *op++ = *ref++;
- ref -= dec[op-ref];
- A32(op)=A32(ref); op += STEPSIZE-4; ref += STEPSIZE-4;
- ref -= dec2;
- } else { LZ4_COPYSTEP(ref,op); }
- cpy = op + length - (STEPSIZE-4);
- if (cpy>oend-COPYLENGTH)
- {
- if (cpy > oend) goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH));
- while(op<cpy) *op++=*ref++;
- op=cpy;
- if (op == oend) break; // Check EOF (should never happen, since last 5 bytes are supposed to be literals)
- continue;
- }
- LZ4_SECURECOPY(ref, op, cpy);
- op=cpy; // correction
- }
-
- // end of decoding
- return (int) (((char*)op)-dest);
-
- // write overflow error detected
-_output_error:
- return (int) (-(((char*)ip)-source));
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.h
deleted file mode 100644
index 8a7f469..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011, Yann Collet.
- BSD License
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-//****************************
-// Simple Functions
-//****************************
-
-int LZ4_compress (char* source, char* dest, int isize);
-int LZ4_uncompress (char* source, char* dest, int osize);
-
-/*
-LZ4_compress() :
- return : the number of bytes in compressed buffer dest
- note : destination buffer must be already allocated.
- To avoid any problem, size it to handle worst cases situations (input data not compressible)
- Worst case size is : "inputsize + 0.4%", with "0.4%" being at least 8 bytes.
-
-LZ4_uncompress() :
- osize : is the output size, therefore the original size
- return : the number of bytes read in the source buffer
- If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction
- This version never writes beyond dest + osize, and is therefore protected against malicious data packets
- note 2 : destination buffer must be already allocated
-*/
-
-
-//****************************
-// Advanced Functions
-//****************************
-
-int LZ4_uncompress_unknownOutputSize (char* source, char* dest, int isize, int maxOutputSize);
-
-/*
-LZ4_uncompress_unknownOutputSize() :
- isize : is the input size, therefore the compressed size
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
- If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction
- This version never writes beyond dest + maxOutputSize, and is therefore protected against malicious data packets
- note : This version is a bit slower than LZ4_uncompress
-*/
-
-
-int LZ4_compressCtx(void** ctx, char* source, char* dest, int isize);
-
-/*
-LZ4_compressCtx() :
- This function explicitly handles the CTX memory structure.
- It avoids allocating/deallocating memory between each call, improving performance when malloc is time-consuming.
- Note : when memory is allocated into the stack (default mode), there is no "malloc" penalty.
- Therefore, this function is mostly useful when memory is allocated into the heap (it requires increasing HASH_LOG value beyond STACK_LIMIT)
-
- On first call : provide a *ctx=NULL; It will be automatically allocated.
- On next calls : reuse the same ctx pointer.
- Use different pointers for different threads when doing multi-threading.
-
- note : performance difference is small, mostly noticeable in HeapMode when repetitively calling the compression function over many small segments.
-*/
-
-
-#if defined (__cplusplus)
-}
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
index b0e9441e..8dc4c64 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
@@ -17,33 +17,15 @@
*/
#include "commons.h"
+#include "lz4.h"
#include "NativeTask.h"
#include "Lz4Codec.h"
-extern "C" {
-extern int LZ4_compress(char* source, char* dest, int isize);
-extern int LZ4_uncompress(char* source, char* dest, int osize);
-
-/*
- LZ4_compress() :
- return : the number of bytes in compressed buffer dest
- note : destination buffer must be already allocated.
- To avoid any problem, size it to handle worst cases situations (input data not compressible)
- Worst case size is : "inputsize + 0.4%", with "0.4%" being at least 8 bytes.
-
- LZ4_uncompress() :
- osize : is the output size, therefore the original size
- return : the number of bytes read in the source buffer
- If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction
- This version never writes beyond dest + osize, and is therefore protected against malicious data packets
- note 2 : destination buffer must be already allocated
- */
-}
namespace NativeTask {
static int32_t LZ4_MaxCompressedSize(int32_t orig) {
- return std::max((int32_t)(orig * 1.005), orig + 8);
+ return LZ4_compressBound(orig);
}
Lz4CompressStream::Lz4CompressStream(OutputStream * stream, uint32_t bufferSizeHint)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
index c65d3d0..68e8368 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+#include "lz4.h"
#include "snappy.h"
#include "commons.h"
#include "Path.h"
@@ -172,11 +173,6 @@ public:
}
};
-extern "C" {
-extern int LZ4_compress(char* source, char* dest, int isize);
-extern int LZ4_uncompress(char* source, char* dest, int osize);
-}
-;
void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t blockSize,
int times) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 07a1d63..22885b2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -162,7 +162,7 @@ TEST(Perf, IFile) {
// The Glibc has a bug in the file tell api, it will overwrite the file data
// unexpected.
// Please check https://rhn.redhat.com/errata/RHBA-2013-0279.html
-// This case is to check wether the bug exists.
+// This case is to check whether the bug exists.
// If it exists, it means you need to upgrade the glibc.
TEST(IFile, TestGlibCBug) {
std::string path("./testData/testGlibCBugSpill.out");
[49/50] [abbrv] git commit: Merge MR-2841 changes into main MapReduce
CHANGES.txt file
Posted by ar...@apache.org.
Merge MR-2841 changes into main MapReduce CHANGES.txt file
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c918172
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c918172
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c918172
Branch: refs/heads/HDFS-6581
Commit: 3c9181722b05a9192f5440ea8f3f77231f84eac6
Parents: 8a3a327
Author: Todd Lipcon <to...@apache.org>
Authored: Fri Sep 12 18:35:13 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 12 18:35:13 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 26 --------
hadoop-mapreduce-project/CHANGES.txt | 65 ++++++++++++++++++++
2 files changed, 65 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c918172/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
deleted file mode 100644
index cfc9412..0000000
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-Changes for Hadoop Native Map Output Collector
-------------------------------------------------
-
-MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
-MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
-MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd)
-MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
-MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
-MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
-MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
-MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
-MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
-MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
-MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
-MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd)
-MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
-MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd)
-MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
-MAPREDUCE-6054. native-task: Speed up tests (todd)
-MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
-MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
-MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
-MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
-MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd)
-MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd)
-MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c918172/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index f1435d2..5d1e5f5 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -20,6 +20,9 @@ Trunk (Unreleased)
MAPREDUCE-5910. Make MR AM resync with RM in case of work-preserving
RM-restart. (Rohith via jianhe)
+ MAPREDUCE-2841. Add a native implementation of MapOutputCollector.
+ (see section below for detailed breakdown)
+
IMPROVEMENTS
MAPREDUCE-3481. [Gridmix] Improve Gridmix STRESS mode. (amarrk)
@@ -157,6 +160,68 @@ Trunk (Unreleased)
MAPREDUCE-5972. Fix typo 'programatically' in job.xml (and a few other
places) (Akira AJISAKA via aw)
+ BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
+
+ MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
+ Binglin Chang
+
+ MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
+
+ MAPREDUCE-5996. native-task: Rename system tests into standard directory
+ layout (todd)
+
+ MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
+
+ MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
+
+ MAPREDUCE-5991. native-task should not run unit tests if native profile is
+ not enabled. (Binglin Chang)
+
+ MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
+
+ MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
+
+ MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin
+ Chang)
+
+ MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
+ (Manu Zhang)
+
+ MAPREDUCE-5978. native-task: remove test case for not supported codec
+ Bzip2Codec and DefaultCodec (Manu Zhang)
+
+ MAPREDUCE-6006. native-task: add native tests to maven and fix bug in
+ pom.xml (Binglin Chang via todd)
+
+ MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
+
+ MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu
+ Zhang via todd)
+
+ MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via
+ todd)
+
+ MAPREDUCE-6054. native-task: Speed up tests (todd)
+
+ MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is
+ sucessful (Binglin Chang)
+
+ MAPREDUCE-6056. native-task: move system test working dir to target dir and
+ cleanup test config xml files (Manu Zhang via bchang)
+
+ MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc
+ cleanup (todd)
+
+ MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
+
+ MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code
+ (todd)
+
+ MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings
+ (todd)
+
+ MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
[04/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into
standard directory layout. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
deleted file mode 100644
index b665971..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-
-public class ResultVerifier {
- /**
- * verify the result
- *
- * @param sample
- * :nativetask output
- * @param source
- * :yuanwenjian
- * @throws Exception
- */
- public static boolean verify(String sample, String source) throws Exception {
- FSDataInputStream sourcein = null;
- FSDataInputStream samplein = null;
-
- final Configuration conf = new Configuration();
- final FileSystem fs = FileSystem.get(conf);
- final Path hdfssource = new Path(source);
- final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource));
-
- final Path hdfssample = new Path(sample);
- final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample));
- if (sourcepaths == null) {
- throw new Exception("source file can not be found");
- }
- if (samplepaths == null) {
- throw new Exception("sample file can not be found");
- }
- if (sourcepaths.length != samplepaths.length) {
- return false;
- }
- for (int i = 0; i < sourcepaths.length; i++) {
- final Path sourcepath = sourcepaths[i];
- // op result file start with "part-r" like part-r-00000
-
- if (!sourcepath.getName().startsWith("part-r")) {
- continue;
- }
- Path samplepath = null;
- for (int j = 0; j < samplepaths.length; j++) {
- if (samplepaths[i].getName().equals(sourcepath.getName())) {
- samplepath = samplepaths[i];
- break;
- }
- }
- if (samplepath == null) {
- throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName()
- + " , as sourcepaths has such file");
- }
-
- // compare
- try {
- if (fs.exists(sourcepath) && fs.exists(samplepath)) {
- sourcein = fs.open(sourcepath);
- samplein = fs.open(samplepath);
- } else {
- System.err.println("result file not found:" + sourcepath + " or " + samplepath);
- return false;
- }
-
- CRC32 sourcecrc, samplecrc;
- samplecrc = new CRC32();
- sourcecrc = new CRC32();
- final byte[] bufin = new byte[1 << 16];
- int readnum = 0;
- int totalRead = 0;
- while (samplein.available() > 0) {
- readnum = samplein.read(bufin);
- totalRead += readnum;
- samplecrc.update(bufin, 0, readnum);
- }
-
- if (0 == totalRead) {
- throw new Exception("source " + sample + " is empty file");
- }
-
- totalRead = 0;
- while (sourcein.available() > 0) {
- readnum = sourcein.read(bufin);
- totalRead += readnum;
- sourcecrc.update(bufin, 0, readnum);
- }
- if (0 == totalRead) {
- throw new Exception("source " + sample + " is empty file");
- }
-
- if (samplecrc.getValue() == sourcecrc.getValue()) {
- ;
- } else {
- return false;
- }
- } catch (final IOException e) {
- throw new Exception("verify exception :", e);
- } finally {
-
- try {
- if (samplein != null) {
- samplein.close();
- }
- if (sourcein != null) {
- sourcein.close();
- }
- } catch (final IOException e) {
- e.printStackTrace();
- }
-
- }
- }
- return true;
- }
-
- public static void main(String[] args) {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
deleted file mode 100644
index 9cc473d..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class ScenarioConfiguration extends Configuration {
- public ScenarioConfiguration() {
- super();
- this.addResource(TestConstants.COMMON_CONF_PATH);
- }
-
- public void addcombinerConf() {
- this.addResource(TestConstants.COMBINER_CONF_PATH);
- }
-
- public void addKVTestConf() {
- this.addResource(TestConstants.KVTEST_CONF_PATH);
- }
-
- public void addNonSortTestConf() {
- this.addResource(TestConstants.NONSORT_TEST_CONF);
- }
-
- public void addNativeConf() {
- this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR,
- TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS);
- }
-
- public static Configuration getNormalConfiguration() {
- Configuration normalConf = new Configuration();
- normalConf.addResource("common_conf.xml");
- normalConf.addResource("normal_conf.xml");
- return normalConf;
- }
-
- public static Configuration getNativeConfiguration() {
- Configuration nativeConf = new Configuration();
- nativeConf.addResource("common_conf.xml");
- nativeConf.addResource("native_conf.xml");
- return nativeConf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
deleted file mode 100644
index 3beb082..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-public class TestConstants {
- // conf path
- public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
- public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
- public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
-
- public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
- public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
-
- public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
- public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
- public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
- public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
- public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
- public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
- public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
- public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
- public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
-
- public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
- public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
- public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
- public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
- public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
- public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
-
- public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
- public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
- public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
- public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
- public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
- public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
- public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
- public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
- public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
- public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
-
- public static final String COMMON_CONF_PATH = "common_conf.xml";
-
- public static final String FILESIZE_KEY = "kvtest.file.size";
- public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
-
- public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
- public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
- public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
- public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
-
-}
[41/50] [abbrv] git commit: MAPREDUCE-6077. native-task: Remove
CustomModule examples in nativetask (seanzhong)
Posted by ar...@apache.org.
MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52a8b4db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52a8b4db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52a8b4db
Branch: refs/heads/HDFS-6581
Commit: 52a8b4db92a35598006103c59a2ff93afc5312ee
Parents: 7c91f9b
Author: Sean Zhong <cl...@gmail.com>
Authored: Sat Sep 6 11:46:07 2014 +0800
Committer: Sean Zhong <cl...@gmail.com>
Committed: Sat Sep 6 11:46:07 2014 +0800
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../sdk/example/CustomModule/README.txt | 1 -
.../sdk/example/CustomModule/pom.xml | 131 -------------------
.../platform/custom/CustomPlatform.java | 39 ------
.../nativetask/serde/custom/CustomWritable.java | 75 -----------
.../serde/custom/CustomWritableSerializer.java | 33 -----
.../src/main/native/src/CustomComparator.cpp | 88 -------------
7 files changed, 1 insertion(+), 367 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 539e7be..cfc9412 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -23,3 +23,4 @@ MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cle
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd)
MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd)
+MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt
deleted file mode 100644
index 0ad6f1e..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt
+++ /dev/null
@@ -1 +0,0 @@
-This project is depend on hadoop and hadoop-nativetask, so in order to complete the build, you have to specify the hadoop dir first.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml
deleted file mode 100644
index 5bf67c7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml
+++ /dev/null
@@ -1,131 +0,0 @@
-<?xml version="1.0"?>
-<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
- may not use this file except in compliance with the License. You may obtain
- a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
- required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License.
- See accompanying LICENSE file. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>nativetask-sdk</artifactId>
- <version>2.2.0</version>
- <name>nativetask-sdk</name>
-
- <properties>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
- <version>2.2.0</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>native</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce-os</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireOS>
- <family>mac</family>
- <family>unix</family>
- <message>native build only supported on Mac or
- Unix</message>
- </requireOS>
- </rules>
- <fail>true</fail>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>make</id>
- <phase>compile</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <exec executable="sh" dir="${basedir}/src/main/native"
- failonerror="true">
- <arg value="${basedir}/src/main/native/build.sh" />
- <arg value="all" />
- </exec>
- <copy toDir="${project.build.directory}/native/target/usr/local/lib" overwrite="true">
- <fileset dir="${basedir}/src/main/native/.libs">
- <exclude name="**/*.o"/>
- <exclude name="nttest"/>
- </fileset>
- </copy>
- <copy todir="${project.build.directory}/${project.artifactId}-${project.version}/lib/native/sdk/" overwrite="true">
- <fileset dir="${basedir}/src/main/native/.libs">
- <exclude name="**/*.o"/>
- <exclude name="nttest"/>
- </fileset>
- <fileset file="${basedir}/src/main/native/src/NativeTask.h" />
- </copy>
- </target>
- </configuration>
- </execution>
- <execution>
- <id>make-clean</id>
- <phase>clean</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <exec executable="sh" dir="${basedir}/src/main/native"
- failonerror="true">
- <arg value="${basedir}/src/main/native/build.sh" />
- <arg value="clean" />
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java
deleted file mode 100644
index 1d67f43..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.nativetask.platform.custom;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.nativetask.Platform;
-import org.apache.hadoop.nativetask.serde.custom.CustomWritable;
-import org.apache.hadoop.nativetask.serde.custom.CustomWritableSerializer;
-
-public class CustomPlatform extends Platform{
-
- @Override
- public void init() throws IOException {
- registerKey(CustomWritable.class.getName(),
- CustomWritableSerializer.class);
- }
-
- @Override
- public String name() {
- return "CustomPlatform";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java
deleted file mode 100644
index 28162f9..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.nativetask.serde.custom;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-public class CustomWritable implements WritableComparable<CustomWritable>{
-
- private int Id_a;
- private long Id_b;
-
- public CustomWritable() {
- this.Id_a = 0;
- this.Id_b = 0;
- }
-
- public CustomWritable(int a, long b){
- this.Id_a = a;
- this.Id_b = b;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- Id_a = in.readInt();
- Id_b = in.readLong();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(Id_a);
- out.writeLong(Id_b);
- }
-
- @Override
- public int compareTo(CustomWritable that) {
- if(Id_a > that.Id_a){
- return 1;
- }
- if(Id_a < that.Id_a){
- return -1;
- }
- if(Id_b > that.Id_b){
- return 1;
- }
- if(Id_b < that.Id_b){
- return -1;
- }
- return 0;
- }
-
- @Override
- public String toString() {
- return Id_a + "\t" + Id_b;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java
deleted file mode 100644
index 4424a2e..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.nativetask.serde.custom;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.nativetask.INativeComparable;
-import org.apache.hadoop.mapred.nativetask.serde.DefaultSerializer;
-
-public class CustomWritableSerializer extends DefaultSerializer implements
- INativeComparable{
-
- @Override
- public int getLength(Writable w) throws IOException {
- return 12;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp
deleted file mode 100644
index c8aa21d..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <iostream>
-#include "NativeTask.h"
-
-using namespace NativeTask;
-
-namespace Custom {
-using namespace std;
-
-inline uint32_t bswap(uint32_t val) {
- __asm__("bswap %0" : "=r" (val) : "0" (val));
- return val;
-}
-
-int32_t ReadInt(const char * src) {
- return (int32_t) bswap(*(uint32_t*) src);
-}
-
-inline uint64_t bswap64(uint64_t val) {
-#ifdef __X64
- __asm__("bswapq %0" : "=r" (val) : "0" (val));
-#else
-
- uint64_t lower = val & 0xffffffffU;
- uint32_t higher = (val >> 32) & 0xffffffffU;
-
- lower = bswap(lower);
- higher = bswap(higher);
-
- return (lower << 32) + higher;
-
-#endif
- return val;
-}
-
-int64_t ReadLong(const char * src) {
- return (int64_t) bswap64(*(uint64_t*) src);
-}
-
-int CustomComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- int32_t src_IDa = ReadInt(src);
- int64_t src_IDb = ReadLong(src+4);
- int32_t dest_IDa = ReadInt(dest);
- int64_t dest_IDb = ReadLong(dest+4);
- if(src_IDa > dest_IDa){
- return 1;
- }
- if(src_IDa < dest_IDa){
- return -1;
- }
- if(src_IDb > dest_IDb){
- return 1;
- }
- if(src_IDb < dest_IDb){
- return -1;
- }
- return 0;
-};
-
-DEFINE_NATIVE_LIBRARY(Custom) {
- REGISTER_FUNCTION(CustomComparator,Custom);
-}
-
-}
-
-
-
-
-
-
[19/50] [abbrv] git commit: MAPREDUCE-6006. native-task: add native
tests to maven and fix bug in pom.xml. Contributed by Binglin Chang.
Posted by ar...@apache.org.
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml. Contributed by Binglin Chang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617877 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/886338ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/886338ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/886338ff
Branch: refs/heads/HDFS-6581
Commit: 886338ff80438b2daea3755f165cc411e39ef560
Parents: 4d4fb17
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Aug 14 04:47:20 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Aug 14 04:47:20 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 3 +-
.../hadoop-mapreduce-client-common/pom.xml | 11 --------
.../hadoop-mapreduce-client-nativetask/pom.xml | 16 +++++++++--
.../src/CMakeLists.txt | 4 +++
.../src/main/native/test.sh | 29 ++++++++++++--------
hadoop-project/pom.xml | 6 ----
6 files changed, 38 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 462ac8b..c3c6ab9 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -11,4 +11,5 @@ MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
-MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
\ No newline at end of file
+MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
+MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 48db5c8..7ef7d3b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -84,17 +84,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 9727800..bb7d7bb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -66,7 +66,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
@@ -173,7 +172,6 @@
overwrite="true">
<fileset dir="${basedir}/src/main/native/testData" />
</copy>
- <copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" />
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}" />
@@ -187,6 +185,20 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>native_tests</id>
+ <phase>test</phase>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/test">
+ <arg value="-c"/>
+ <arg value="[ x$SKIPTESTS = xtrue ] || sh test.sh"/>
+ <env key="SKIPTESTS" value="${skipTests}"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index f38021d..77b6109 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -167,10 +167,14 @@ SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
# macosx does not have -lrt
set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+ set(SYSTEM_MAC TRUE)
else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+ set(SYSTEM_MAC FALSE)
endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+configure_file(main/native/test.sh test/test.sh)
+
add_dual_library(nativetask
${CMAKE_BINARY_DIR}/lz4.c
${D}/cityhash/city.cc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
index 7310c9d..01506e6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test.sh
@@ -1,18 +1,25 @@
-#!/bin/bash
+#!/bin/sh
+
+# only do normal tests by default
+FILTER="--gtest_filter=-Perf.*"
# do all tests
-if [ "$1" == "all" ]; then
-shift
-./nttest $@
-exit $?
+if [ "$1" = "all" ]; then
+ shift
+ FILTER=""
fi
# do performance tests only
-if [ "$1" == "perf" ]; then
-shift
-./nttest --gtest_filter=Perf.* $@
-exit $?
+if [ "$1" = "perf" ]; then
+ shift
+ FILTER="--gtest_filter=Perf.*"
+fi
+
+if [ "${SYSTEM_MAC}" = "TRUE" ]; then
+ # MACOSX already setup RPATH, no extra help required
+ ./nttest $FILTER $@
+else
+ JAVA_JVM_LIBRARY_DIR=`dirname ${JAVA_JVM_LIBRARY}`
+ LD_LIBRARY_PATH=$JAVA_JVM_LIBRARY_DIR:$LD_LIBRARY_PATH ./nttest $FILTER $@
fi
-# do not do performance test by default
-./nttest --gtest_filter=-Perf.* $@
http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 947be86..2dc83dd 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -136,12 +136,6 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${project.version}</version>
</dependency>
[24/50] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc
warnings. Contributed by Manu Zhang.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
new file mode 100644
index 0000000..b467822
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
@@ -0,0 +1,19541 @@
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// The Google C++ Testing Framework (Google Test)
+//
+// This header file defines the public API for Google Test. It should be
+// included by any test program that uses Google Test.
+//
+// IMPORTANT NOTE: Due to limitation of the C++ language, we have to
+// leave some internal implementation details in this header file.
+// They are clearly marked by comments like this:
+//
+// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM.
+//
+// Such code is NOT meant to be used by a user directly, and is subject
+// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user
+// program!
+//
+// Acknowledgment: Google Test borrowed the idea of automatic test
+// registration from Barthelemy Dagenais' (barthelemy@prologique.com)
+// easyUnit framework.
+
+#ifndef GTEST_INCLUDE_GTEST_GTEST_H_
+#define GTEST_INCLUDE_GTEST_GTEST_H_
+
+#include <limits>
+#include <vector>
+
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee)
+//
+// The Google C++ Testing Framework (Google Test)
+//
+// This header file declares functions and macros used internally by
+// Google Test. They are subject to change without notice.
+
+#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_
+#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_
+
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: wan@google.com (Zhanyong Wan)
+//
+// Low-level types and utilities for porting Google Test to various
+// platforms. They are subject to change without notice. DO NOT USE
+// THEM IN USER CODE.
+
+#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
+#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
+
+// The user can define the following macros in the build script to
+// control Google Test's behavior. If the user doesn't define a macro
+// in this list, Google Test will define it.
+//
+// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2)
+// is/isn't available.
+// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions
+// are enabled.
+// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string
+// is/isn't available (some systems define
+// ::string, which is different to std::string).
+// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::string
+// is/isn't available (some systems define
+// ::wstring, which is different to std::wstring).
+// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular
+// expressions are/aren't available.
+// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that <pthread.h>
+// is/isn't available.
+// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't
+// enabled.
+// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that
+// std::wstring does/doesn't work (Google Test can
+// be used where std::wstring is unavailable).
+// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple
+// is/isn't available.
+// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the
+// compiler supports Microsoft's "Structured
+// Exception Handling".
+// GTEST_HAS_STREAM_REDIRECTION
+// - Define it to 1/0 to indicate whether the
+// platform supports I/O stream redirection using
+// dup() and dup2().
+// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google
+// Test's own tr1 tuple implementation should be
+// used. Unused when the user sets
+// GTEST_HAS_TR1_TUPLE to 0.
+// GTEST_LINKED_AS_SHARED_LIBRARY
+// - Define to 1 when compiling tests that use
+// Google Test as a shared library (known as
+// DLL on Windows).
+// GTEST_CREATE_SHARED_LIBRARY
+// - Define to 1 when compiling Google Test itself
+// as a shared library.
+
+// This header defines the following utilities:
+//
+// Macros indicating the current platform (defined to 1 if compiled on
+// the given platform; otherwise undefined):
+// GTEST_OS_AIX - IBM AIX
+// GTEST_OS_CYGWIN - Cygwin
+// GTEST_OS_HPUX - HP-UX
+// GTEST_OS_LINUX - Linux
+// GTEST_OS_LINUX_ANDROID - Google Android
+// GTEST_OS_MAC - Mac OS X
+// GTEST_OS_NACL - Google Native Client (NaCl)
+// GTEST_OS_SOLARIS - Sun Solaris
+// GTEST_OS_SYMBIAN - Symbian
+// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile)
+// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop
+// GTEST_OS_WINDOWS_MINGW - MinGW
+// GTEST_OS_WINDOWS_MOBILE - Windows Mobile
+// GTEST_OS_ZOS - z/OS
+//
+// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the
+// most stable support. Since core members of the Google Test project
+// don't have access to other platforms, support for them may be less
+// stable. If you notice any problems on your platform, please notify
+// googletestframework@googlegroups.com (patches for fixing them are
+// even more welcome!).
+//
+// Note that it is possible that none of the GTEST_OS_* macros are defined.
+//
+// Macros indicating available Google Test features (defined to 1 if
+// the corresponding feature is supported; otherwise undefined):
+// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized
+// tests)
+// GTEST_HAS_DEATH_TEST - death tests
+// GTEST_HAS_PARAM_TEST - value-parameterized tests
+// GTEST_HAS_TYPED_TEST - typed tests
+// GTEST_HAS_TYPED_TEST_P - type-parameterized tests
+// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with
+// GTEST_HAS_POSIX_RE (see above) which users can
+// define themselves.
+// GTEST_USES_SIMPLE_RE - our own simple regex is used;
+// the above two are mutually exclusive.
+// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ().
+//
+// Macros for basic C++ coding:
+// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning.
+// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a
+// variable don't have to be used.
+// GTEST_DISALLOW_ASSIGN_ - disables operator=.
+// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=.
+// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used.
+//
+// Synchronization:
+// Mutex, MutexLock, ThreadLocal, GetThreadCount()
+// - synchronization primitives.
+// GTEST_IS_THREADSAFE - defined to 1 to indicate that the above
+// synchronization primitives have real implementations
+// and Google Test is thread-safe; or 0 otherwise.
+//
+// Template meta programming:
+// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only.
+// IteratorTraits - partial implementation of std::iterator_traits, which
+// is not available in libCstd when compiled with Sun C++.
+//
+// Smart pointers:
+// scoped_ptr - as in TR2.
+//
+// Regular expressions:
+// RE - a simple regular expression class using the POSIX
+// Extended Regular Expression syntax on UNIX-like
+// platforms, or a reduced regular exception syntax on
+// other platforms, including Windows.
+//
+// Logging:
+// GTEST_LOG_() - logs messages at the specified severity level.
+// LogToStderr() - directs all log messages to stderr.
+// FlushInfoLog() - flushes informational log messages.
+//
+// Stdout and stderr capturing:
+// CaptureStdout() - starts capturing stdout.
+// GetCapturedStdout() - stops capturing stdout and returns the captured
+// string.
+// CaptureStderr() - starts capturing stderr.
+// GetCapturedStderr() - stops capturing stderr and returns the captured
+// string.
+//
+// Integer types:
+// TypeWithSize - maps an integer to a int type.
+// Int32, UInt32, Int64, UInt64, TimeInMillis
+// - integers of known sizes.
+// BiggestInt - the biggest signed integer type.
+//
+// Command-line utilities:
+// GTEST_FLAG() - references a flag.
+// GTEST_DECLARE_*() - declares a flag.
+// GTEST_DEFINE_*() - defines a flag.
+// GetArgvs() - returns the command line as a vector of strings.
+//
+// Environment variable utilities:
+// GetEnv() - gets the value of an environment variable.
+// BoolFromGTestEnv() - parses a bool environment variable.
+// Int32FromGTestEnv() - parses an Int32 environment variable.
+// StringFromGTestEnv() - parses a string environment variable.
+
+#include <ctype.h> // for isspace, etc
+#include <stddef.h> // for ptrdiff_t
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#ifndef _WIN32_WCE
+# include <sys/types.h>
+# include <sys/stat.h>
+#endif // !_WIN32_WCE
+
+#include <iostream> // NOLINT
+#include <sstream> // NOLINT
+#include <string> // NOLINT
+
+#define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com"
+#define GTEST_FLAG_PREFIX_ "gtest_"
+#define GTEST_FLAG_PREFIX_DASH_ "gtest-"
+#define GTEST_FLAG_PREFIX_UPPER_ "GTEST_"
+#define GTEST_NAME_ "Google Test"
+#define GTEST_PROJECT_URL_ "http://code.google.com/p/googletest/"
+
+// Determines the version of gcc that is used to compile this.
+#ifdef __GNUC__
+// 40302 means version 4.3.2.
+# define GTEST_GCC_VER_ \
+ (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__)
+#endif // __GNUC__
+
+// Determines the platform on which Google Test is compiled.
+#ifdef __CYGWIN__
+# define GTEST_OS_CYGWIN 1
+#elif defined __SYMBIAN32__
+# define GTEST_OS_SYMBIAN 1
+#elif defined _WIN32
+# define GTEST_OS_WINDOWS 1
+# ifdef _WIN32_WCE
+# define GTEST_OS_WINDOWS_MOBILE 1
+# elif defined(__MINGW__) || defined(__MINGW32__)
+# define GTEST_OS_WINDOWS_MINGW 1
+# else
+# define GTEST_OS_WINDOWS_DESKTOP 1
+# endif // _WIN32_WCE
+#elif defined __APPLE__
+# define GTEST_OS_MAC 1
+#elif defined __linux__
+# define GTEST_OS_LINUX 1
+# ifdef ANDROID
+# define GTEST_OS_LINUX_ANDROID 1
+# endif // ANDROID
+#elif defined __MVS__
+# define GTEST_OS_ZOS 1
+#elif defined(__sun) && defined(__SVR4)
+# define GTEST_OS_SOLARIS 1
+#elif defined(_AIX)
+# define GTEST_OS_AIX 1
+#elif defined(__hpux)
+# define GTEST_OS_HPUX 1
+#elif defined __native_client__
+# define GTEST_OS_NACL 1
+#endif // __CYGWIN__
+
+// Brings in definitions for functions used in the testing::internal::posix
+// namespace (read, write, close, chdir, isatty, stat). We do not currently
+// use them on Windows Mobile.
+#if !GTEST_OS_WINDOWS
+// This assumes that non-Windows OSes provide unistd.h. For OSes where this
+// is not the case, we need to include headers that provide the functions
+// mentioned above.
+# include <unistd.h>
+# if !GTEST_OS_NACL
+// TODO(vladl@google.com): Remove this condition when Native Client SDK adds
+// strings.h (tracked in
+// http://code.google.com/p/nativeclient/issues/detail?id=1175).
+# include <strings.h> // Native Client doesn't provide strings.h.
+# endif
+#elif !GTEST_OS_WINDOWS_MOBILE
+# include <direct.h>
+# include <io.h>
+#endif
+
+// Defines this to true iff Google Test can use POSIX regular expressions.
+#ifndef GTEST_HAS_POSIX_RE
+# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS)
+#endif
+
+#if GTEST_HAS_POSIX_RE
+
+// On some platforms, <regex.h> needs someone to define size_t, and
+// won't compile otherwise. We can #include it here as we already
+// included <stdlib.h>, which is guaranteed to define size_t through
+// <stddef.h>.
+# include <regex.h> // NOLINT
+
+# define GTEST_USES_POSIX_RE 1
+
+#elif GTEST_OS_WINDOWS
+
+// <regex.h> is not available on Windows. Use our own simple regex
+// implementation instead.
+# define GTEST_USES_SIMPLE_RE 1
+
+#else
+
+// <regex.h> may not be available on this platform. Use our own
+// simple regex implementation instead.
+# define GTEST_USES_SIMPLE_RE 1
+
+#endif // GTEST_HAS_POSIX_RE
+
+#ifndef GTEST_HAS_EXCEPTIONS
+// The user didn't tell us whether exceptions are enabled, so we need
+// to figure it out.
+# if defined(_MSC_VER) || defined(__BORLANDC__)
+// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS
+// macro to enable exceptions, so we'll do the same.
+// Assumes that exceptions are enabled by default.
+# ifndef _HAS_EXCEPTIONS
+# define _HAS_EXCEPTIONS 1
+# endif // _HAS_EXCEPTIONS
+# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS
+# elif defined(__GNUC__) && __EXCEPTIONS
+// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled.
+# define GTEST_HAS_EXCEPTIONS 1
+# elif defined(__SUNPRO_CC)
+// Sun Pro CC supports exceptions. However, there is no compile-time way of
+// detecting whether they are enabled or not. Therefore, we assume that
+// they are enabled unless the user tells us otherwise.
+# define GTEST_HAS_EXCEPTIONS 1
+# elif defined(__IBMCPP__) && __EXCEPTIONS
+// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled.
+# define GTEST_HAS_EXCEPTIONS 1
+# elif defined(__HP_aCC)
+// Exception handling is in effect by default in HP aCC compiler. It has to
+// be turned of by +noeh compiler option if desired.
+# define GTEST_HAS_EXCEPTIONS 1
+# else
+// For other compilers, we assume exceptions are disabled to be
+// conservative.
+# define GTEST_HAS_EXCEPTIONS 0
+# endif // defined(_MSC_VER) || defined(__BORLANDC__)
+#endif // GTEST_HAS_EXCEPTIONS
+
+#if !defined(GTEST_HAS_STD_STRING)
+// Even though we don't use this macro any longer, we keep it in case
+// some clients still depend on it.
+# define GTEST_HAS_STD_STRING 1
+#elif !GTEST_HAS_STD_STRING
+// The user told us that ::std::string isn't available.
+# error "Google Test cannot be used where ::std::string isn't available."
+#endif // !defined(GTEST_HAS_STD_STRING)
+
+#ifndef GTEST_HAS_GLOBAL_STRING
+// The user didn't tell us whether ::string is available, so we need
+// to figure it out.
+
+# define GTEST_HAS_GLOBAL_STRING 0
+
+#endif // GTEST_HAS_GLOBAL_STRING
+
+#ifndef GTEST_HAS_STD_WSTRING
+// The user didn't tell us whether ::std::wstring is available, so we need
+// to figure it out.
+// TODO(wan@google.com): uses autoconf to detect whether ::std::wstring
+// is available.
+
+// Cygwin 1.7 and below doesn't support ::std::wstring.
+// Solaris' libc++ doesn't support it either. Android has
+// no support for it at least as recent as Froyo (2.2).
+# define GTEST_HAS_STD_WSTRING \
+ (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS))
+
+#endif // GTEST_HAS_STD_WSTRING
+
+#ifndef GTEST_HAS_GLOBAL_WSTRING
+// The user didn't tell us whether ::wstring is available, so we need
+// to figure it out.
+# define GTEST_HAS_GLOBAL_WSTRING \
+ (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING)
+#endif // GTEST_HAS_GLOBAL_WSTRING
+
+// Determines whether RTTI is available.
+#ifndef GTEST_HAS_RTTI
+// The user didn't tell us whether RTTI is enabled, so we need to
+// figure it out.
+
+# ifdef _MSC_VER
+
+# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled.
+# define GTEST_HAS_RTTI 1
+# else
+# define GTEST_HAS_RTTI 0
+# endif
+
+// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled.
+# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302)
+
+# ifdef __GXX_RTTI
+# define GTEST_HAS_RTTI 1
+# else
+# define GTEST_HAS_RTTI 0
+# endif // __GXX_RTTI
+
+// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if
+// both the typeid and dynamic_cast features are present.
+# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900)
+
+# ifdef __RTTI_ALL__
+# define GTEST_HAS_RTTI 1
+# else
+# define GTEST_HAS_RTTI 0
+# endif
+
+# else
+
+// For all other compilers, we assume RTTI is enabled.
+# define GTEST_HAS_RTTI 1
+
+# endif // _MSC_VER
+
+#endif // GTEST_HAS_RTTI
+
+// It's this header's responsibility to #include <typeinfo> when RTTI
+// is enabled.
+#if GTEST_HAS_RTTI
+# include <typeinfo>
+#endif
+
+// Determines whether Google Test can use the pthreads library.
+#ifndef GTEST_HAS_PTHREAD
+// The user didn't tell us explicitly, so we assume pthreads support is
+// available on Linux and Mac.
+//
+// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0
+// to your compiler flags.
+# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX)
+#endif // GTEST_HAS_PTHREAD
+
+#if GTEST_HAS_PTHREAD
+// gtest-port.h guarantees to #include <pthread.h> when GTEST_HAS_PTHREAD is
+// true.
+# include <pthread.h> // NOLINT
+
+// For timespec and nanosleep, used below.
+# include <time.h> // NOLINT
+#endif
+
+// Determines whether Google Test can use tr1/tuple. You can define
+// this macro to 0 to prevent Google Test from using tuple (any
+// feature depending on tuple with be disabled in this mode).
+#ifndef GTEST_HAS_TR1_TUPLE
+// The user didn't tell us not to do it, so we assume it's OK.
+# define GTEST_HAS_TR1_TUPLE 1
+#endif // GTEST_HAS_TR1_TUPLE
+
+// Determines whether Google Test's own tr1 tuple implementation
+// should be used.
+#ifndef GTEST_USE_OWN_TR1_TUPLE
+// The user didn't tell us, so we need to figure it out.
+
+// We use our own TR1 tuple if we aren't sure the user has an
+// implementation of it already. At this time, GCC 4.0.0+ and MSVC
+// 2010 are the only mainstream compilers that come with a TR1 tuple
+// implementation. NVIDIA's CUDA NVCC compiler pretends to be GCC by
+// defining __GNUC__ and friends, but cannot compile GCC's tuple
+// implementation. MSVC 2008 (9.0) provides TR1 tuple in a 323 MB
+// Feature Pack download, which we cannot assume the user has.
+# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000)) \
+ || _MSC_VER >= 1600
+# define GTEST_USE_OWN_TR1_TUPLE 0
+# else
+# define GTEST_USE_OWN_TR1_TUPLE 1
+# endif
+
+#endif // GTEST_USE_OWN_TR1_TUPLE
+
+// To avoid conditional compilation everywhere, we make it
+// gtest-port.h's responsibility to #include the header implementing
+// tr1/tuple.
+#if GTEST_HAS_TR1_TUPLE
+
+# if GTEST_USE_OWN_TR1_TUPLE
+// This file was GENERATED by a script. DO NOT EDIT BY HAND!!!
+
+// Copyright 2009 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Implements a subset of TR1 tuple needed by Google Test and Google Mock.
+
+#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
+#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
+
+#include <utility> // For ::std::pair.
+
+// The compiler used in Symbian has a bug that prevents us from declaring the
+// tuple template as a friend (it complains that tuple is redefined). This
+// hack bypasses the bug by declaring the members that should otherwise be
+// private as public.
+// Sun Studio versions < 12 also have the above bug.
+#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590)
+# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public:
+#else
+# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \
+ template <GTEST_10_TYPENAMES_(U)> friend class tuple; \
+ private:
+#endif
+
+// GTEST_n_TUPLE_(T) is the type of an n-tuple.
+#define GTEST_0_TUPLE_(T) tuple<>
+#define GTEST_1_TUPLE_(T) tuple<T##0, void, void, void, void, void, void, \
+ void, void, void>
+#define GTEST_2_TUPLE_(T) tuple<T##0, T##1, void, void, void, void, void, \
+ void, void, void>
+#define GTEST_3_TUPLE_(T) tuple<T##0, T##1, T##2, void, void, void, void, \
+ void, void, void>
+#define GTEST_4_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, void, void, void, \
+ void, void, void>
+#define GTEST_5_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, void, void, \
+ void, void, void>
+#define GTEST_6_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, void, \
+ void, void, void>
+#define GTEST_7_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
+ void, void, void>
+#define GTEST_8_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
+ T##7, void, void>
+#define GTEST_9_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
+ T##7, T##8, void>
+#define GTEST_10_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
+ T##7, T##8, T##9>
+
+// GTEST_n_TYPENAMES_(T) declares a list of n typenames.
+#define GTEST_0_TYPENAMES_(T)
+#define GTEST_1_TYPENAMES_(T) typename T##0
+#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1
+#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2
+#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3
+#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4
+#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4, typename T##5
+#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4, typename T##5, typename T##6
+#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4, typename T##5, typename T##6, typename T##7
+#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4, typename T##5, typename T##6, \
+ typename T##7, typename T##8
+#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
+ typename T##3, typename T##4, typename T##5, typename T##6, \
+ typename T##7, typename T##8, typename T##9
+
+// In theory, defining stuff in the ::std namespace is undefined
+// behavior. We can do this as we are playing the role of a standard
+// library vendor.
+namespace std {
+namespace tr1 {
+
+template <typename T0 = void, typename T1 = void, typename T2 = void,
+ typename T3 = void, typename T4 = void, typename T5 = void,
+ typename T6 = void, typename T7 = void, typename T8 = void,
+ typename T9 = void>
+class tuple;
+
+// Anything in namespace gtest_internal is Google Test's INTERNAL
+// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code.
+namespace gtest_internal {
+
+// ByRef<T>::type is T if T is a reference; otherwise it's const T&.
+template <typename T>
+struct ByRef { typedef const T& type; }; // NOLINT
+template <typename T>
+struct ByRef<T&> { typedef T& type; }; // NOLINT
+
+// A handy wrapper for ByRef.
+#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef<T>::type
+
+// AddRef<T>::type is T if T is a reference; otherwise it's T&. This
+// is the same as tr1::add_reference<T>::type.
+template <typename T>
+struct AddRef { typedef T& type; }; // NOLINT
+template <typename T>
+struct AddRef<T&> { typedef T& type; }; // NOLINT
+
+// A handy wrapper for AddRef.
+#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef<T>::type
+
+// A helper for implementing get<k>().
+template <int k> class Get;
+
+// A helper for implementing tuple_element<k, T>. kIndexValid is true
+// iff k < the number of fields in tuple type T.
+template <bool kIndexValid, int kIndex, class Tuple>
+struct TupleElement;
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 0, GTEST_10_TUPLE_(T)> { typedef T0 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 1, GTEST_10_TUPLE_(T)> { typedef T1 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 2, GTEST_10_TUPLE_(T)> { typedef T2 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 3, GTEST_10_TUPLE_(T)> { typedef T3 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 4, GTEST_10_TUPLE_(T)> { typedef T4 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 5, GTEST_10_TUPLE_(T)> { typedef T5 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 6, GTEST_10_TUPLE_(T)> { typedef T6 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 7, GTEST_10_TUPLE_(T)> { typedef T7 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 8, GTEST_10_TUPLE_(T)> { typedef T8 type; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct TupleElement<true, 9, GTEST_10_TUPLE_(T)> { typedef T9 type; };
+
+} // namespace gtest_internal
+
+template <>
+class tuple<> {
+ public:
+ tuple() {}
+ tuple(const tuple& /* t */) {}
+ tuple& operator=(const tuple& /* t */) { return *this; }
+};
+
+template <GTEST_1_TYPENAMES_(T)>
+class GTEST_1_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {}
+
+ tuple(const tuple& t) : f0_(t.f0_) {}
+
+ template <GTEST_1_TYPENAMES_(U)>
+ tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_1_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_1_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_1_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ return *this;
+ }
+
+ T0 f0_;
+};
+
+template <GTEST_2_TYPENAMES_(T)>
+class GTEST_2_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0),
+ f1_(f1) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {}
+
+ template <GTEST_2_TYPENAMES_(U)>
+ tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {}
+ template <typename U0, typename U1>
+ tuple(const ::std::pair<U0, U1>& p) : f0_(p.first), f1_(p.second) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_2_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_2_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+ template <typename U0, typename U1>
+ tuple& operator=(const ::std::pair<U0, U1>& p) {
+ f0_ = p.first;
+ f1_ = p.second;
+ return *this;
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_2_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+};
+
+template <GTEST_3_TYPENAMES_(T)>
+class GTEST_3_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {}
+
+ template <GTEST_3_TYPENAMES_(U)>
+ tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_3_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_3_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_3_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+};
+
+template <GTEST_4_TYPENAMES_(T)>
+class GTEST_4_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2),
+ f3_(f3) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {}
+
+ template <GTEST_4_TYPENAMES_(U)>
+ tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_4_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_4_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_4_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+};
+
+template <GTEST_5_TYPENAMES_(T)>
+class GTEST_5_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3,
+ GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_) {}
+
+ template <GTEST_5_TYPENAMES_(U)>
+ tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_5_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_5_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_5_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+};
+
+template <GTEST_6_TYPENAMES_(T)>
+class GTEST_6_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
+ GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
+ f5_(f5) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_), f5_(t.f5_) {}
+
+ template <GTEST_6_TYPENAMES_(U)>
+ tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_6_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_6_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_6_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ f5_ = t.f5_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+ T5 f5_;
+};
+
+template <GTEST_7_TYPENAMES_(T)>
+class GTEST_7_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
+ GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2),
+ f3_(f3), f4_(f4), f5_(f5), f6_(f6) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {}
+
+ template <GTEST_7_TYPENAMES_(U)>
+ tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_7_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_7_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_7_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ f5_ = t.f5_;
+ f6_ = t.f6_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+ T5 f5_;
+ T6 f6_;
+};
+
+template <GTEST_8_TYPENAMES_(T)>
+class GTEST_8_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
+ GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6,
+ GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
+ f5_(f5), f6_(f6), f7_(f7) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {}
+
+ template <GTEST_8_TYPENAMES_(U)>
+ tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_8_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_8_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_8_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ f5_ = t.f5_;
+ f6_ = t.f6_;
+ f7_ = t.f7_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+ T5 f5_;
+ T6 f6_;
+ T7 f7_;
+};
+
+template <GTEST_9_TYPENAMES_(T)>
+class GTEST_9_TUPLE_(T) {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
+ GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7,
+ GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
+ f5_(f5), f6_(f6), f7_(f7), f8_(f8) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {}
+
+ template <GTEST_9_TYPENAMES_(U)>
+ tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_9_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_9_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_9_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ f5_ = t.f5_;
+ f6_ = t.f6_;
+ f7_ = t.f7_;
+ f8_ = t.f8_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+ T5 f5_;
+ T6 f6_;
+ T7 f7_;
+ T8 f8_;
+};
+
+template <GTEST_10_TYPENAMES_(T)>
+class tuple {
+ public:
+ template <int k> friend class gtest_internal::Get;
+
+ tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(),
+ f9_() {}
+
+ explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
+ GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
+ GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7,
+ GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2),
+ f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {}
+
+ tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
+ f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {}
+
+ template <GTEST_10_TYPENAMES_(U)>
+ tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
+ f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_),
+ f9_(t.f9_) {}
+
+ tuple& operator=(const tuple& t) { return CopyFrom(t); }
+
+ template <GTEST_10_TYPENAMES_(U)>
+ tuple& operator=(const GTEST_10_TUPLE_(U)& t) {
+ return CopyFrom(t);
+ }
+
+ GTEST_DECLARE_TUPLE_AS_FRIEND_
+
+ template <GTEST_10_TYPENAMES_(U)>
+ tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) {
+ f0_ = t.f0_;
+ f1_ = t.f1_;
+ f2_ = t.f2_;
+ f3_ = t.f3_;
+ f4_ = t.f4_;
+ f5_ = t.f5_;
+ f6_ = t.f6_;
+ f7_ = t.f7_;
+ f8_ = t.f8_;
+ f9_ = t.f9_;
+ return *this;
+ }
+
+ T0 f0_;
+ T1 f1_;
+ T2 f2_;
+ T3 f3_;
+ T4 f4_;
+ T5 f5_;
+ T6 f6_;
+ T7 f7_;
+ T8 f8_;
+ T9 f9_;
+};
+
+// 6.1.3.2 Tuple creation functions.
+
+// Known limitations: we don't support passing an
+// std::tr1::reference_wrapper<T> to make_tuple(). And we don't
+// implement tie().
+
+inline tuple<> make_tuple() { return tuple<>(); }
+
+template <GTEST_1_TYPENAMES_(T)>
+inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) {
+ return GTEST_1_TUPLE_(T)(f0);
+}
+
+template <GTEST_2_TYPENAMES_(T)>
+inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) {
+ return GTEST_2_TUPLE_(T)(f0, f1);
+}
+
+template <GTEST_3_TYPENAMES_(T)>
+inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) {
+ return GTEST_3_TUPLE_(T)(f0, f1, f2);
+}
+
+template <GTEST_4_TYPENAMES_(T)>
+inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3) {
+ return GTEST_4_TUPLE_(T)(f0, f1, f2, f3);
+}
+
+template <GTEST_5_TYPENAMES_(T)>
+inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4) {
+ return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4);
+}
+
+template <GTEST_6_TYPENAMES_(T)>
+inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4, const T5& f5) {
+ return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5);
+}
+
+template <GTEST_7_TYPENAMES_(T)>
+inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4, const T5& f5, const T6& f6) {
+ return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6);
+}
+
+template <GTEST_8_TYPENAMES_(T)>
+inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) {
+ return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7);
+}
+
+template <GTEST_9_TYPENAMES_(T)>
+inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7,
+ const T8& f8) {
+ return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8);
+}
+
+template <GTEST_10_TYPENAMES_(T)>
+inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
+ const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7,
+ const T8& f8, const T9& f9) {
+ return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9);
+}
+
+// 6.1.3.3 Tuple helper classes.
+
+template <typename Tuple> struct tuple_size;
+
+template <GTEST_0_TYPENAMES_(T)>
+struct tuple_size<GTEST_0_TUPLE_(T)> { static const int value = 0; };
+
+template <GTEST_1_TYPENAMES_(T)>
+struct tuple_size<GTEST_1_TUPLE_(T)> { static const int value = 1; };
+
+template <GTEST_2_TYPENAMES_(T)>
+struct tuple_size<GTEST_2_TUPLE_(T)> { static const int value = 2; };
+
+template <GTEST_3_TYPENAMES_(T)>
+struct tuple_size<GTEST_3_TUPLE_(T)> { static const int value = 3; };
+
+template <GTEST_4_TYPENAMES_(T)>
+struct tuple_size<GTEST_4_TUPLE_(T)> { static const int value = 4; };
+
+template <GTEST_5_TYPENAMES_(T)>
+struct tuple_size<GTEST_5_TUPLE_(T)> { static const int value = 5; };
+
+template <GTEST_6_TYPENAMES_(T)>
+struct tuple_size<GTEST_6_TUPLE_(T)> { static const int value = 6; };
+
+template <GTEST_7_TYPENAMES_(T)>
+struct tuple_size<GTEST_7_TUPLE_(T)> { static const int value = 7; };
+
+template <GTEST_8_TYPENAMES_(T)>
+struct tuple_size<GTEST_8_TUPLE_(T)> { static const int value = 8; };
+
+template <GTEST_9_TYPENAMES_(T)>
+struct tuple_size<GTEST_9_TUPLE_(T)> { static const int value = 9; };
+
+template <GTEST_10_TYPENAMES_(T)>
+struct tuple_size<GTEST_10_TUPLE_(T)> { static const int value = 10; };
+
+template <int k, class Tuple>
+struct tuple_element {
+ typedef typename gtest_internal::TupleElement<
+ k < (tuple_size<Tuple>::value), k, Tuple>::type type;
+};
+
+#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element<k, Tuple >::type
+
+// 6.1.3.4 Element access.
+
+namespace gtest_internal {
+
+template <>
+class Get<0> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple))
+ Field(Tuple& t) { return t.f0_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple))
+ ConstField(const Tuple& t) { return t.f0_; }
+};
+
+template <>
+class Get<1> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple))
+ Field(Tuple& t) { return t.f1_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple))
+ ConstField(const Tuple& t) { return t.f1_; }
+};
+
+template <>
+class Get<2> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple))
+ Field(Tuple& t) { return t.f2_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple))
+ ConstField(const Tuple& t) { return t.f2_; }
+};
+
+template <>
+class Get<3> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple))
+ Field(Tuple& t) { return t.f3_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple))
+ ConstField(const Tuple& t) { return t.f3_; }
+};
+
+template <>
+class Get<4> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple))
+ Field(Tuple& t) { return t.f4_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple))
+ ConstField(const Tuple& t) { return t.f4_; }
+};
+
+template <>
+class Get<5> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple))
+ Field(Tuple& t) { return t.f5_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple))
+ ConstField(const Tuple& t) { return t.f5_; }
+};
+
+template <>
+class Get<6> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple))
+ Field(Tuple& t) { return t.f6_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple))
+ ConstField(const Tuple& t) { return t.f6_; }
+};
+
+template <>
+class Get<7> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple))
+ Field(Tuple& t) { return t.f7_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple))
+ ConstField(const Tuple& t) { return t.f7_; }
+};
+
+template <>
+class Get<8> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple))
+ Field(Tuple& t) { return t.f8_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple))
+ ConstField(const Tuple& t) { return t.f8_; }
+};
+
+template <>
+class Get<9> {
+ public:
+ template <class Tuple>
+ static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple))
+ Field(Tuple& t) { return t.f9_; } // NOLINT
+
+ template <class Tuple>
+ static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple))
+ ConstField(const Tuple& t) { return t.f9_; }
+};
+
+} // namespace gtest_internal
+
+template <int k, GTEST_10_TYPENAMES_(T)>
+GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T)))
+get(GTEST_10_TUPLE_(T)& t) {
+ return gtest_internal::Get<k>::Field(t);
+}
+
+template <int k, GTEST_10_TYPENAMES_(T)>
+GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T)))
+get(const GTEST_10_TUPLE_(T)& t) {
+ return gtest_internal::Get<k>::ConstField(t);
+}
+
+// 6.1.3.5 Relational operators
+
+// We only implement == and !=, as we don't have a need for the rest yet.
+
+namespace gtest_internal {
+
+// SameSizeTuplePrefixComparator<k, k>::Eq(t1, t2) returns true if the
+// first k fields of t1 equals the first k fields of t2.
+// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if
+// k1 != k2.
+template <int kSize1, int kSize2>
+struct SameSizeTuplePrefixComparator;
+
+template <>
+struct SameSizeTuplePrefixComparator<0, 0> {
+ template <class Tuple1, class Tuple2>
+ static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) {
+ return true;
+ }
+};
+
+template <int k>
+struct SameSizeTuplePrefixComparator<k, k> {
+ template <class Tuple1, class Tuple2>
+ static bool Eq(const Tuple1& t1, const Tuple2& t2) {
+ return SameSizeTuplePrefixComparator<k - 1, k - 1>::Eq(t1, t2) &&
+ ::std::tr1::get<k - 1>(t1) == ::std::tr1::get<k - 1>(t2);
+ }
+};
+
+} // namespace gtest_internal
+
+template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)>
+inline bool operator==(const GTEST_10_TUPLE_(T)& t,
+ const GTEST_10_TUPLE_(U)& u) {
+ return gtest_internal::SameSizeTuplePrefixComparator<
+ tuple_size<GTEST_10_TUPLE_(T)>::value,
+ tuple_size<GTEST_10_TUPLE_(U)>::value>::Eq(t, u);
+}
+
+template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)>
+inline bool operator!=(const GTEST_10_TUPLE_(T)& t,
+ const GTEST_10_TUPLE_(U)& u) { return !(t == u); }
+
+// 6.1.4 Pairs.
+// Unimplemented.
+
+} // namespace tr1
+} // namespace std
+
+#undef GTEST_0_TUPLE_
+#undef GTEST_1_TUPLE_
+#undef GTEST_2_TUPLE_
+#undef GTEST_3_TUPLE_
+#undef GTEST_4_TUPLE_
+#undef GTEST_5_TUPLE_
+#undef GTEST_6_TUPLE_
+#undef GTEST_7_TUPLE_
+#undef GTEST_8_TUPLE_
+#undef GTEST_9_TUPLE_
+#undef GTEST_10_TUPLE_
+
+#undef GTEST_0_TYPENAMES_
+#undef GTEST_1_TYPENAMES_
+#undef GTEST_2_TYPENAMES_
+#undef GTEST_3_TYPENAMES_
+#undef GTEST_4_TYPENAMES_
+#undef GTEST_5_TYPENAMES_
+#undef GTEST_6_TYPENAMES_
+#undef GTEST_7_TYPENAMES_
+#undef GTEST_8_TYPENAMES_
+#undef GTEST_9_TYPENAMES_
+#undef GTEST_10_TYPENAMES_
+
+#undef GTEST_DECLARE_TUPLE_AS_FRIEND_
+#undef GTEST_BY_REF_
+#undef GTEST_ADD_REF_
+#undef GTEST_TUPLE_ELEMENT_
+
+#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
+# elif GTEST_OS_SYMBIAN
+
+// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to
+// use STLport's tuple implementation, which unfortunately doesn't
+// work as the copy of STLport distributed with Symbian is incomplete.
+// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to
+// use its own tuple implementation.
+# ifdef BOOST_HAS_TR1_TUPLE
+# undef BOOST_HAS_TR1_TUPLE
+# endif // BOOST_HAS_TR1_TUPLE
+
+// This prevents <boost/tr1/detail/config.hpp>, which defines
+// BOOST_HAS_TR1_TUPLE, from being #included by Boost's <tuple>.
+# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED
+# include <tuple>
+
+# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000)
+// GCC 4.0+ implements tr1/tuple in the <tr1/tuple> header. This does
+// not conform to the TR1 spec, which requires the header to be <tuple>.
+
+# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302
+// Until version 4.3.2, gcc has a bug that causes <tr1/functional>,
+// which is #included by <tr1/tuple>, to not compile when RTTI is
+// disabled. _TR1_FUNCTIONAL is the header guard for
+// <tr1/functional>. Hence the following #define is a hack to prevent
+// <tr1/functional> from being included.
+# define _TR1_FUNCTIONAL 1
+# include <tr1/tuple>
+# undef _TR1_FUNCTIONAL // Allows the user to #include
+ // <tr1/functional> if he chooses to.
+# else
+# include <tr1/tuple> // NOLINT
+# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302
+
+# else
+// If the compiler is not GCC 4.0+, we assume the user is using a
+// spec-conforming TR1 implementation.
+# include <tuple> // NOLINT
+# endif // GTEST_USE_OWN_TR1_TUPLE
+
+#endif // GTEST_HAS_TR1_TUPLE
+
+// Determines whether clone(2) is supported.
+// Usually it will only be available on Linux, excluding
+// Linux on the Itanium architecture.
+// Also see http://linux.die.net/man/2/clone.
+#ifndef GTEST_HAS_CLONE
+// The user didn't tell us, so we need to figure it out.
+
+# if GTEST_OS_LINUX && !defined(__ia64__)
+# define GTEST_HAS_CLONE 1
+# else
+# define GTEST_HAS_CLONE 0
+# endif // GTEST_OS_LINUX && !defined(__ia64__)
+
+#endif // GTEST_HAS_CLONE
+
+// Determines whether to support stream redirection. This is used to test
+// output correctness and to implement death tests.
+#ifndef GTEST_HAS_STREAM_REDIRECTION
+// By default, we assume that stream redirection is supported on all
+// platforms except known mobile ones.
+# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN
+# define GTEST_HAS_STREAM_REDIRECTION 0
+# else
+# define GTEST_HAS_STREAM_REDIRECTION 1
+# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN
+#endif // GTEST_HAS_STREAM_REDIRECTION
+
+// Determines whether to support death tests.
+// Google Test does not support death tests for VC 7.1 and earlier as
+// abort() in a VC 7.1 application compiled as GUI in debug config
+// pops up a dialog window that cannot be suppressed programmatically.
+#if (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \
+ (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \
+ GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX)
+# define GTEST_HAS_DEATH_TEST 1
+# include <vector> // NOLINT
+#endif
+
+// We don't support MSVC 7.1 with exceptions disabled now. Therefore
+// all the compilers we care about are adequate for supporting
+// value-parameterized tests.
+#define GTEST_HAS_PARAM_TEST 1
+
+// Determines whether to support type-driven tests.
+
+// Typed tests need <typeinfo> and variadic macros, which GCC, VC++ 8.0,
+// Sun Pro CC, IBM Visual Age, and HP aCC support.
+#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \
+ defined(__IBMCPP__) || defined(__HP_aCC)
+# define GTEST_HAS_TYPED_TEST 1
+# define GTEST_HAS_TYPED_TEST_P 1
+#endif
+
+// Determines whether to support Combine(). This only makes sense when
+// value-parameterized tests are enabled. The implementation doesn't
+// work on Sun Studio since it doesn't understand templated conversion
+// operators.
+#if GTEST_HAS_PARAM_TEST && GTEST_HAS_TR1_TUPLE && !defined(__SUNPRO_CC)
+# define GTEST_HAS_COMBINE 1
+#endif
+
+// Determines whether the system compiler uses UTF-16 for encoding wide strings.
+#define GTEST_WIDE_STRING_USES_UTF16_ \
+ (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX)
+
+// Determines whether test results can be streamed to a socket.
+#if GTEST_OS_LINUX
+# define GTEST_CAN_STREAM_RESULTS_ 1
+#endif
+
+// Defines some utility macros.
+
+// The GNU compiler emits a warning if nested "if" statements are followed by
+// an "else" statement and braces are not used to explicitly disambiguate the
+// "else" binding. This leads to problems with code like:
+//
+// if (gate)
+// ASSERT_*(condition) << "Some message";
+//
+// The "switch (0) case 0:" idiom is used to suppress this.
+#ifdef __INTEL_COMPILER
+# define GTEST_AMBIGUOUS_ELSE_BLOCKER_
+#else
+# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT
+#endif
+
+// Use this annotation at the end of a struct/class definition to
+// prevent the compiler from optimizing away instances that are never
+// used. This is useful when all interesting logic happens inside the
+// c'tor and / or d'tor. Example:
+//
+// struct Foo {
+// Foo() { ... }
+// } GTEST_ATTRIBUTE_UNUSED_;
+//
+// Also use it after a variable or parameter declaration to tell the
+// compiler the variable/parameter does not have to be used.
+#if defined(__GNUC__) && !defined(COMPILER_ICC)
+# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused))
+#else
+# define GTEST_ATTRIBUTE_UNUSED_
+#endif
+
+// A macro to disallow operator=
+// This should be used in the private: declarations for a class.
+#define GTEST_DISALLOW_ASSIGN_(type)\
+ void operator=(type const &)
+
+// A macro to disallow copy constructor and operator=
+// This should be used in the private: declarations for a class.
+#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type)\
+ type(type const &);\
+ GTEST_DISALLOW_ASSIGN_(type)
+
+// Tell the compiler to warn about unused return values for functions declared
+// with this macro. The macro should be used on function declarations
+// following the argument list:
+//
+// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_;
+#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC)
+# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result))
+#else
+# define GTEST_MUST_USE_RESULT_
+#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC
+
+// Determine whether the compiler supports Microsoft's Structured Exception
+// Handling. This is supported by several Windows compilers but generally
+// does not exist on any other system.
+#ifndef GTEST_HAS_SEH
+// The user didn't tell us, so we need to figure it out.
+
+# if defined(_MSC_VER) || defined(__BORLANDC__)
+// These two compilers are known to support SEH.
+# define GTEST_HAS_SEH 1
+# else
+// Assume no SEH.
+# define GTEST_HAS_SEH 0
+# endif
+
+#endif // GTEST_HAS_SEH
+
+#ifdef _MSC_VER
+
+# if GTEST_LINKED_AS_SHARED_LIBRARY
+# define GTEST_API_ __declspec(dllimport)
+# elif GTEST_CREATE_SHARED_LIBRARY
+# define GTEST_API_ __declspec(dllexport)
+# endif
+
+#endif // _MSC_VER
+
+#ifndef GTEST_API_
+# define GTEST_API_
+#endif
+
+#ifdef __GNUC__
+// Ask the compiler to never inline a given function.
+# define GTEST_NO_INLINE_ __attribute__((noinline))
+#else
+# define GTEST_NO_INLINE_
+#endif
+
+namespace testing {
+
+class Message;
+
+namespace internal {
+
+class String;
+
+// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time
+// expression is true. For example, you could use it to verify the
+// size of a static array:
+//
+// GTEST_COMPILE_ASSERT_(ARRAYSIZE(content_type_names) == CONTENT_NUM_TYPES,
+// content_type_names_incorrect_size);
+//
+// or to make sure a struct is smaller than a certain size:
+//
+// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large);
+//
+// The second argument to the macro is the name of the variable. If
+// the expression is false, most compilers will issue a warning/error
+// containing the name of the variable.
+
+template <bool>
+struct CompileAssert {
+};
+
+#define GTEST_COMPILE_ASSERT_(expr, msg) \
+ typedef ::testing::internal::CompileAssert<(bool(expr))> \
+ msg[bool(expr) ? 1 : -1]
+
+// Implementation details of GTEST_COMPILE_ASSERT_:
+//
+// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1
+// elements (and thus is invalid) when the expression is false.
+//
+// - The simpler definition
+//
+// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1]
+//
+// does not work, as gcc supports variable-length arrays whose sizes
+// are determined at run-time (this is gcc's extension and not part
+// of the C++ standard). As a result, gcc fails to reject the
+// following code with the simple definition:
+//
+// int foo;
+// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is
+// // not a compile-time constant.
+//
+// - By using the type CompileAssert<(bool(expr))>, we ensures that
+// expr is a compile-time constant. (Template arguments must be
+// determined at compile-time.)
+//
+// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
+// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written
+//
+// CompileAssert<bool(expr)>
+//
+// instead, these compilers will refuse to compile
+//
+// GTEST_COMPILE_ASSERT_(5 > 0, some_message);
+//
+// (They seem to think the ">" in "5 > 0" marks the end of the
+// template argument list.)
+//
+// - The array size is (bool(expr) ? 1 : -1), instead of simply
+//
+// ((expr) ? 1 : -1).
+//
+// This is to avoid running into a bug in MS VC 7.1, which
+// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
+
+// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h.
+//
+// This template is declared, but intentionally undefined.
+template <typename T1, typename T2>
+struct StaticAssertTypeEqHelper;
+
+template <typename T>
+struct StaticAssertTypeEqHelper<T, T> {};
+
+#if GTEST_HAS_GLOBAL_STRING
+typedef ::string string;
+#else
+typedef ::std::string string;
+#endif // GTEST_HAS_GLOBAL_STRING
+
+#if GTEST_HAS_GLOBAL_WSTRING
+typedef ::wstring wstring;
+#elif GTEST_HAS_STD_WSTRING
+typedef ::std::wstring wstring;
+#endif // GTEST_HAS_GLOBAL_WSTRING
+
+// A helper for suppressing warnings on constant condition. It just
+// returns 'condition'.
+GTEST_API_ bool IsTrue(bool condition);
+
+// Defines scoped_ptr.
+
+// This implementation of scoped_ptr is PARTIAL - it only contains
+// enough stuff to satisfy Google Test's need.
+template <typename T>
+class scoped_ptr {
+ public:
+ typedef T element_type;
+
+ explicit scoped_ptr(T* p = NULL) : ptr_(p) {}
+ ~scoped_ptr() { reset(); }
+
+ T& operator*() const { return *ptr_; }
+ T* operator->() const { return ptr_; }
+ T* get() const { return ptr_; }
+
+ T* release() {
+ T* const ptr = ptr_;
+ ptr_ = NULL;
+ return ptr;
+ }
+
+ void reset(T* p = NULL) {
+ if (p != ptr_) {
+ if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type.
+ delete ptr_;
+ }
+ ptr_ = p;
+ }
+ }
+ private:
+ T* ptr_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr);
+};
+
+// Defines RE.
+
+// A simple C++ wrapper for <regex.h>. It uses the POSIX Extended
+// Regular Expression syntax.
+class GTEST_API_ RE {
+ public:
+ // A copy constructor is required by the Standard to initialize object
+ // references from r-values.
+ RE(const RE& other) { Init(other.pattern()); }
+
+ // Constructs an RE from a string.
+ RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT
+
+#if GTEST_HAS_GLOBAL_STRING
+
+ RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT
+
+#endif // GTEST_HAS_GLOBAL_STRING
+
+ RE(const char* regex) { Init(regex); } // NOLINT
+ ~RE();
+
+ // Returns the string representation of the regex.
+ const char* pattern() const { return pattern_; }
+
+ // FullMatch(str, re) returns true iff regular expression re matches
+ // the entire str.
+ // PartialMatch(str, re) returns true iff regular expression re
+ // matches a substring of str (including str itself).
+ //
+ // TODO(wan@google.com): make FullMatch() and PartialMatch() work
+ // when str contains NUL characters.
+ static bool FullMatch(const ::std::string& str, const RE& re) {
+ return FullMatch(str.c_str(), re);
+ }
+ static bool PartialMatch(const ::std::string& str, const RE& re) {
+ return PartialMatch(str.c_str(), re);
+ }
+
+#if GTEST_HAS_GLOBAL_STRING
+
+ static bool FullMatch(const ::string& str, const RE& re) {
+ return FullMatch(str.c_str(), re);
+ }
+ static bool PartialMatch(const ::string& str, const RE& re) {
+ return PartialMatch(str.c_str(), re);
+ }
+
+#endif // GTEST_HAS_GLOBAL_STRING
+
+ static bool FullMatch(const char* str, const RE& re);
+ static bool PartialMatch(const char* str, const RE& re);
+
+ private:
+ void Init(const char* regex);
+
+ // We use a const char* instead of a string, as Google Test may be used
+ // where string is not available. We also do not use Google Test's own
+ // String type here, in order to simplify dependencies between the
+ // files.
+ const char* pattern_;
+ bool is_valid_;
+
+#if GTEST_USES_POSIX_RE
+
+ regex_t full_regex_; // For FullMatch().
+ regex_t partial_regex_; // For PartialMatch().
+
+#else // GTEST_USES_SIMPLE_RE
+
+ const char* full_pattern_; // For FullMatch();
+
+#endif
+
+ GTEST_DISALLOW_ASSIGN_(RE);
+};
+
+// Formats a source file path and a line number as they would appear
+// in an error message from the compiler used to compile this code.
+GTEST_API_ ::std::string FormatFileLocation(const char* file, int line);
+
+// Formats a file location for compiler-independent XML output.
+// Although this function is not platform dependent, we put it next to
+// FormatFileLocation in order to contrast the two functions.
+GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file,
+ int line);
+
+// Defines logging utilities:
+// GTEST_LOG_(severity) - logs messages at the specified severity level. The
+// message itself is streamed into the macro.
+// LogToStderr() - directs all log messages to stderr.
+// FlushInfoLog() - flushes informational log messages.
+
+enum GTestLogSeverity {
+ GTEST_INFO,
+ GTEST_WARNING,
+ GTEST_ERROR,
+ GTEST_FATAL
+};
+
+// Formats log entry severity, provides a stream object for streaming the
+// log message, and terminates the message with a newline when going out of
+// scope.
+class GTEST_API_ GTestLog {
+ public:
+ GTestLog(GTestLogSeverity severity, const char* file, int line);
+
+ // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program.
+ ~GTestLog();
+
+ ::std::ostream& GetStream() { return ::std::cerr; }
+
+ private:
+ const GTestLogSeverity severity_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog);
+};
+
+#define GTEST_LOG_(severity) \
+ ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \
+ __FILE__, __LINE__).GetStream()
+
+inline void LogToStderr() {}
+inline void FlushInfoLog() { fflush(NULL); }
+
+// INTERNAL IMPLEMENTATION - DO NOT USE.
+//
+// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition
+// is not satisfied.
+// Synopsys:
+// GTEST_CHECK_(boolean_condition);
+// or
+// GTEST_CHECK_(boolean_condition) << "Additional message";
+//
+// This checks the condition and if the condition is not satisfied
+// it prints message about the condition violation, including the
+// condition itself, plus additional message streamed into it, if any,
+// and then it aborts the program. It aborts the program irrespective of
+// whether it is built in the debug mode or not.
+#define GTEST_CHECK_(condition) \
+ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
+ if (::testing::internal::IsTrue(condition)) \
+ ; \
+ else \
+ GTEST_LOG_(FATAL) << "Condition " #condition " failed. "
+
+// An all-mode assert to verify that the given POSIX-style function
+// call returns 0 (indicating success). Known limitation: this
+// doesn't expand to a balanced 'if' statement, so enclose the macro
+// in {} if you need to use it as the only statement in an 'if'
+// branch.
+#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \
+ if (const int gtest_error = (posix_call)) \
+ GTEST_LOG_(FATAL) << #posix_call << "failed with error " \
+ << gtest_error
+
+// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE.
+//
+// Use ImplicitCast_ as a safe version of static_cast for upcasting in
+// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a
+// const Foo*). When you use ImplicitCast_, the compiler checks that
+// the cast is safe. Such explicit ImplicitCast_s are necessary in
+// surprisingly many situations where C++ demands an exact type match
+// instead of an argument type convertable to a target type.
+//
+// The syntax for using ImplicitCast_ is the same as for static_cast:
+//
+// ImplicitCast_<ToType>(expr)
+//
+// ImplicitCast_ would have been part of the C++ standard library,
+// but the proposal was submitted too late. It will probably make
+// its way into the language in the future.
+//
+// This relatively ugly name is intentional. It prevents clashes with
+// similar functions users may have (e.g., implicit_cast). The internal
+// namespace alone is not enough because the function can be found by ADL.
+template<typename To>
+inline To ImplicitCast_(To x) { return x; }
+
+// When you upcast (that is, cast a pointer from type Foo to type
+// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts
+// always succeed. When you downcast (that is, cast a pointer from
+// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because
+// how do you know the pointer is really of type SubclassOfFoo? It
+// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus,
+// when you downcast, you should use this macro. In debug mode, we
+// use dynamic_cast<> to double-check the downcast is legal (we die
+// if it's not). In normal mode, we do the efficient static_cast<>
+// instead. Thus, it's important to test in debug mode to make sure
+// the cast is legal!
+// This is the only place in the code we should use dynamic_cast<>.
+// In particular, you SHOULDN'T be using dynamic_cast<> in order to
+// do RTTI (eg code like this:
+// if (dynamic_cast<Subclass1>(foo)) HandleASubclass1Object(foo);
+// if (dynamic_cast<Subclass2>(foo)) HandleASubclass2Object(foo);
+// You should design the code some other way not to need this.
+//
+// This relatively ugly name is intentional. It prevents clashes with
+// similar functions users may have (e.g., down_cast). The internal
+// namespace alone is not enough because the function can be found by ADL.
+template<typename To, typename From> // use like this: DownCast_<T*>(foo);
+inline To DownCast_(From* f) { // so we only accept pointers
+ // Ensures that To is a sub-type of From *. This test is here only
+ // for compile-time type checking, and has no overhead in an
+ // optimized build at run-time, as it will be optimized away
+ // completely.
+ if (false) {
+ const To to = NULL;
+ ::testing::internal::ImplicitCast_<From*>(to);
+ }
+
+#if GTEST_HAS_RTTI
+ // RTTI: debug mode only!
+ GTEST_CHECK_(f == NULL || dynamic_cast<To>(f) != NULL);
+#endif
+ return static_cast<To>(f);
+}
+
+// Downcasts the pointer of type Base to Derived.
+// Derived must be a subclass of Base. The parameter MUST
+// point to a class of type Derived, not any subclass of it.
+// When RTTI is available, the function performs a runtime
+// check to enforce this.
+template <class Derived, class Base>
+Derived* CheckedDowncastToActualType(Base* base) {
+#if GTEST_HAS_RTTI
+ GTEST_CHECK_(typeid(*base) == typeid(Derived));
+ return dynamic_cast<Derived*>(base); // NOLINT
+#else
+ return static_cast<Derived*>(base); // Poor man's downcast.
+#endif
+}
+
+#if GTEST_HAS_STREAM_REDIRECTION
+
+// Defines the stderr capturer:
+// CaptureStdout - starts capturing stdout.
+// GetCapturedStdout - stops capturing stdout and returns the captured string.
+// CaptureStderr - starts capturing stderr.
+// GetCapturedStderr - stops capturing stderr and returns the captured string.
+//
+GTEST_API_ void CaptureStdout();
+GTEST_API_ String GetCapturedStdout();
+GTEST_API_ void CaptureStderr();
+GTEST_API_ String GetCapturedStderr();
+
+#endif // GTEST_HAS_STREAM_REDIRECTION
+
+
+#if GTEST_HAS_DEATH_TEST
+
+// A copy of all command line arguments. Set by InitGoogleTest().
+extern ::std::vector<String> g_argvs;
+
+// GTEST_HAS_DEATH_TEST implies we have ::std::string.
+const ::std::vector<String>& GetArgvs();
+
+#endif // GTEST_HAS_DEATH_TEST
+
+// Defines synchronization primitives.
+
+#if GTEST_HAS_PTHREAD
+
+// Sleeps for (roughly) n milli-seconds. This function is only for
+// testing Google Test's own constructs. Don't use it in user tests,
+// either directly or indirectly.
+inline void SleepMilliseconds(int n) {
+ const timespec time = {
+ 0, // 0 seconds.
+ n * 1000L * 1000L, // And n ms.
+ };
+ nanosleep(&time, NULL);
+}
+
+// Allows a controller thread to pause execution of newly created
+// threads until notified. Instances of this class must be created
+// and destroyed in the controller thread.
+//
+// This class is only for testing Google Test's own constructs. Do not
+// use it in user tests, either directly or indirectly.
+class Notification {
+ public:
+ Notification() : notified_(false) {}
+
+ // Notifies all threads created with this notification to start. Must
+ // be called from the controller thread.
+ void Notify() { notified_ = true; }
+
+ // Blocks until the controller thread notifies. Must be called from a test
+ // thread.
+ void WaitForNotification() {
+ while(!notified_) {
+ SleepMilliseconds(10);
+ }
+ }
+
+ private:
+ volatile bool notified_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification);
+};
+
+// As a C-function, ThreadFuncWithCLinkage cannot be templated itself.
+// Consequently, it cannot select a correct instantiation of ThreadWithParam
+// in order to call its Run(). Introducing ThreadWithParamBase as a
+// non-templated base class for ThreadWithParam allows us to bypass this
+// problem.
+class ThreadWithParamBase {
+ public:
+ virtual ~ThreadWithParamBase() {}
+ virtual void Run() = 0;
+};
+
+// pthread_create() accepts a pointer to a function type with the C linkage.
+// According to the Standard (7.5/1), function types with different linkages
+// are different even if they are otherwise identical. Some compilers (for
+// example, SunStudio) treat them as different types. Since class methods
+// cannot be defined with C-linkage we need to define a free C-function to
+// pass into pthread_create().
+extern "C" inline void* ThreadFuncWithCLinkage(void* thread) {
+ static_cast<ThreadWithParamBase*>(thread)->Run();
+ return NULL;
+}
+
+// Helper class for testing Google Test's multi-threading constructs.
+// To use it, write:
+//
+// void ThreadFunc(int param) { /* Do things with param */ }
+// Notification thread_can_start;
+// ...
+// // The thread_can_start parameter is optional; you can supply NULL.
+// ThreadWithParam<int> thread(&ThreadFunc, 5, &thread_can_start);
+// thread_can_start.Notify();
+//
+// These classes are only for testing Google Test's own constructs. Do
+// not use them in user tests, either directly or indirectly.
+template <typename T>
+class ThreadWithParam : public ThreadWithParamBase {
+ public:
+ typedef void (*UserThreadFunc)(T);
+
+ ThreadWithParam(
+ UserThreadFunc func, T param, Notification* thread_can_start)
+ : func_(func),
+ param_(param),
+ thread_can_start_(thread_can_start),
+ finished_(false) {
+ ThreadWithParamBase* const base = this;
+ // The thread can be created only after all fields except thread_
+ // have been initialized.
+ GTEST_CHECK_POSIX_SUCCESS_(
+ pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base));
+ }
+ ~ThreadWithParam() { Join(); }
+
+ void Join() {
+ if (!finished_) {
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0));
+ finished_ = true;
+ }
+ }
+
+ virtual void Run() {
+ if (thread_can_start_ != NULL)
+ thread_can_start_->WaitForNotification();
+ func_(param_);
+ }
+
+ private:
+ const UserThreadFunc func_; // User-supplied thread function.
+ const T param_; // User-supplied parameter to the thread function.
+ // When non-NULL, used to block execution until the controller thread
+ // notifies.
+ Notification* const thread_can_start_;
+ bool finished_; // true iff we know that the thread function has finished.
+ pthread_t thread_; // The native thread object.
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam);
+};
+
+// MutexBase and Mutex implement mutex on pthreads-based platforms. They
+// are used in conjunction with class MutexLock:
+//
+// Mutex mutex;
+// ...
+// MutexLock lock(&mutex); // Acquires the mutex and releases it at the end
+// // of the current scope.
+//
+// MutexBase implements behavior for both statically and dynamically
+// allocated mutexes. Do not use MutexBase directly. Instead, write
+// the following to define a static mutex:
+//
+// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex);
+//
+// You can forward declare a static mutex like this:
+//
+// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex);
+//
+// To create a dynamic mutex, just define an object of type Mutex.
+class MutexBase {
+ public:
+ // Acquires this mutex.
+ void Lock() {
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_));
+ owner_ = pthread_self();
+ }
+
+ // Releases this mutex.
+ void Unlock() {
+ // We don't protect writing to owner_ here, as it's the caller's
+ // responsibility to ensure that the current thread holds the
+ // mutex when this is called.
+ owner_ = 0;
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_));
+ }
+
+ // Does nothing if the current thread holds the mutex. Otherwise, crashes
+ // with high probability.
+ void AssertHeld() const {
+ GTEST_CHECK_(owner_ == pthread_self())
+ << "The current thread is not holding the mutex @" << this;
+ }
+
+ // A static mutex may be used before main() is entered. It may even
+ // be used before the dynamic initialization stage. Therefore we
+ // must be able to initialize a static mutex object at link time.
+ // This means MutexBase has to be a POD and its member variables
+ // have to be public.
+ public:
+ pthread_mutex_t mutex_; // The underlying pthread mutex.
+ pthread_t owner_; // The thread holding the mutex; 0 means no one holds it.
+};
+
+// Forward-declares a static mutex.
+# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
+ extern ::testing::internal::MutexBase mutex
+
+// Defines and statically (i.e. at link time) initializes a static mutex.
+# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \
+ ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, 0 }
+
+// The Mutex class can only be used for mutexes created at runtime. It
+// shares its API with MutexBase otherwise.
+class Mutex : public MutexBase {
+ public:
+ Mutex() {
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL));
+ owner_ = 0;
+ }
+ ~Mutex() {
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_));
+ }
+
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex);
+};
+
+// We cannot name this class MutexLock as the ctor declaration would
+// conflict with a macro named MutexLock, which is defined on some
+// platforms. Hence the typedef trick below.
+class GTestMutexLock {
+ public:
+ explicit GTestMutexLock(MutexBase* mutex)
+ : mutex_(mutex) { mutex_->Lock(); }
+
+ ~GTestMutexLock() { mutex_->Unlock(); }
+
+ private:
+ MutexBase* const mutex_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock);
+};
+
+typedef GTestMutexLock MutexLock;
+
+// Helpers for ThreadLocal.
+
+// pthread_key_create() requires DeleteThreadLocalValue() to have
+// C-linkage. Therefore it cannot be templatized to access
+// ThreadLocal<T>. Hence the need for class
+// ThreadLocalValueHolderBase.
+class ThreadLocalValueHolderBase {
+ public:
+ virtual ~ThreadLocalValueHolderBase() {}
+};
+
+// Called by pthread to delete thread-local data stored by
+// pthread_setspecific().
+extern "C" inline void DeleteThreadLocalValue(void* value_holder) {
+ delete static_cast<ThreadLocalValueHolderBase*>(value_holder);
+}
+
+// Implements thread-local storage on pthreads-based systems.
+//
+// // Thread 1
+// ThreadLocal<int> tl(100); // 100 is the default value for each thread.
+//
+// // Thread 2
+// tl.set(150); // Changes the value for thread 2 only.
+// EXPECT_EQ(150, tl.get());
+//
+// // Thread 1
+// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value.
+// tl.set(200);
+// EXPECT_EQ(200, tl.get());
+//
+// The template type argument T must have a public copy constructor.
+// In addition, the default ThreadLocal constructor requires T to have
+// a public default constructor.
+//
+// An object managed for a thread by a ThreadLocal instance is deleted
+// when the thread exits. Or, if the ThreadLocal instance dies in
+// that thread, when the ThreadLocal dies. It's the user's
+// responsibility to ensure that all other threads using a ThreadLocal
+// have exited when it dies, or the per-thread objects for those
+// threads will not be deleted.
+//
+// Google Test only uses global ThreadLocal objects. That means they
+// will die after main() has returned. Therefore, no per-thread
+// object managed by Google Test will be leaked as long as all threads
+// using Google Test have exited when main() returns.
+template <typename T>
+class ThreadLocal {
+ public:
+ ThreadLocal() : key_(CreateKey()),
+ default_() {}
+ explicit ThreadLocal(const T& value) : key_(CreateKey()),
+ default_(value) {}
+
+ ~ThreadLocal() {
+ // Destroys the managed object for the current thread, if any.
+ DeleteThreadLocalValue(pthread_getspecific(key_));
+
+ // Releases resources associated with the key. This will *not*
+ // delete managed objects for other threads.
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_));
+ }
+
+ T* pointer() { return GetOrCreateValue(); }
+ const T* pointer() const { return GetOrCreateValue(); }
+ const T& get() const { return *pointer(); }
+ void set(const T& value) { *pointer() = value; }
+
+ private:
+ // Holds a value of type T.
+ class ValueHolder : public ThreadLocalValueHolderBase {
+ public:
+ explicit ValueHolder(const T& value) : value_(value) {}
+
+ T* pointer() { return &value_; }
+
+ private:
+ T value_;
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder);
+ };
+
+ static pthread_key_t CreateKey() {
+ pthread_key_t key;
+ // When a thread exits, DeleteThreadLocalValue() will be called on
+ // the object managed for that thread.
+ GTEST_CHECK_POSIX_SUCCESS_(
+ pthread_key_create(&key, &DeleteThreadLocalValue));
+ return key;
+ }
+
+ T* GetOrCreateValue() const {
+ ThreadLocalValueHolderBase* const holder =
+ static_cast<ThreadLocalValueHolderBase*>(pthread_getspecific(key_));
+ if (holder != NULL) {
+ return CheckedDowncastToActualType<ValueHolder>(holder)->pointer();
+ }
+
+ ValueHolder* const new_holder = new ValueHolder(default_);
+ ThreadLocalValueHolderBase* const holder_base = new_holder;
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base));
+ return new_holder->pointer();
+ }
+
+ // A key pthreads uses for looking up per-thread values.
+ const pthread_key_t key_;
+ const T default_; // The default value for each thread.
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal);
+};
+
+# define GTEST_IS_THREADSAFE 1
+
+#else // GTEST_HAS_PTHREAD
+
+// A dummy implementation of synchronization primitives (mutex, lock,
+// and thread-local variable). Necessary for compiling Google Test where
+// mutex is not supported - using Google Test in multiple threads is not
+// supported on such platforms.
+
+class Mutex {
+ public:
+ Mutex() {}
+ void AssertHeld() const {}
+};
+
+# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
+ extern ::testing::internal::Mutex mutex
+
+# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex
+
+class GTestMutexLock {
+ public:
+ explicit GTestMutexLock(Mutex*) {} // NOLINT
+};
+
+typedef GTestMutexLock MutexLock;
+
+template <typename T>
+class ThreadLocal {
+ public:
+ ThreadLocal() : value_() {}
+ explicit ThreadLocal(const T& value) : value_(value) {}
+ T* pointer() { return &value_; }
+ const T* pointer() const { return &value_; }
+ const T& get() const { return value_; }
+ void set(const T& value) { value_ = value; }
+ private:
+ T value_;
+};
+
+// The above synchronization primitives have dummy implementations.
+// Therefore Google Test is not thread-safe.
+# define GTEST_IS_THREADSAFE 0
+
+#endif // GTEST_HAS_PTHREAD
+
+// Returns the number of threads running in the process, or 0 to indicate that
+// we cannot detect it.
+GTEST_API_ size_t GetThreadCount();
+
+// Passing non-POD classes through ellipsis (...) crashes the ARM
+// compiler and generates a warning in Sun Studio. The Nokia Symbian
+// and the IBM XL C/C++ compiler try to instantiate a copy constructor
+// for objects passed through ellipsis (...), failing for uncopyable
+// objects. We define this to ensure that only POD is passed through
+// ellipsis on these systems.
+#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || defined(__SUNPRO_CC)
+// We lose support for NULL detection where the compiler doesn't like
+// passing non-POD classes through ellipsis (...).
+# define GTEST_ELLIPSIS_NEEDS_POD_ 1
+#else
+# define GTEST_CAN_COMPARE_NULL 1
+#endif
+
+// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between
+// const T& and const T* in a function template. These compilers
+// _can_ decide between class template specializations for T and T*,
+// so a tr1::type_traits-like is_pointer works.
+#if defined(__SYMBIAN32__) || defined(__IBMCPP__)
+# define GTEST_NEEDS_IS_POINTER_ 1
+#endif
+
+template <bool bool_value>
+struct bool_constant {
+ typedef bool_constant<bool_value> type;
+ static const bool value = bool_value;
+};
+template <bool bool_value> const bool bool_constant<bool_value>::value;
+
+typedef bool_constant<false> false_type;
+typedef bool_constant<true> true_type;
+
+template <typename T>
+struct is_pointer : public false_type {};
+
+template <typename T>
+struct is_pointer<T*> : public true_type {};
+
+template <typename Iterator>
+struct IteratorTraits {
+ typedef typename Iterator::value_type value_type;
+};
+
+template <typename T>
+struct IteratorTraits<T*> {
+ typedef T value_type;
+};
+
+template <typename T>
+struct IteratorTraits<const T*> {
+ typedef T value_type;
+};
+
+#if GTEST_OS_WINDOWS
+# define GTEST_PATH_SEP_ "\\"
+# define GTEST_HAS_ALT_PATH_SEP_ 1
+// The biggest signed integer type the compiler supports.
+typedef __int64 BiggestInt;
+#else
+# define GTEST_PATH_SEP_ "/"
+# define GTEST_HAS_ALT_PATH_SEP_ 0
+typedef long long BiggestInt; // NOLINT
+#endif // GTEST_OS_WINDOWS
+
+// Utilities for char.
+
+// isspace(int ch) and friends accept an unsigned char or EOF. char
+// may be signed, depending on the compiler (or compiler flags).
+// Therefore we need to cast a char to unsigned char before calling
+// isspace(), etc.
+
+inline bool IsAlpha(char ch) {
+ return isalpha(static_cast<unsigned char>(ch)) != 0;
+}
+inline bool IsAlNum(char ch) {
+ return isalnum(static_cast<unsigned char>(ch
<TRUNCATED>
[37/50] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of
unused code. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 60bb6f5..ba026f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -38,13 +38,6 @@ using std::pair;
enum NativeObjectType {
UnknownObjectType = 0,
BatchHandlerType = 1,
- MapperType = 2,
- ReducerType = 3,
- PartitionerType = 4,
- CombinerType = 5,
- FolderType = 6,
- RecordReaderType = 7,
- RecordWriterType = 8
};
/**
@@ -69,7 +62,6 @@ enum Endium {
#define INPUT_LINE_KV_SEPERATOR "mapreduce.input.keyvaluelinerecordreader.key.value.separator"
#define MAPRED_TEXTOUTPUT_FORMAT_SEPERATOR "mapreduce.output.textoutputformat.separator"
#define MAPRED_WORK_OUT_DIR "mapreduce.task.output.dir"
-#define NATIVE_OUTPUT_FILE_NAME "native.output.file.name"
#define MAPRED_COMPRESS_OUTPUT "mapreduce.output.fileoutputformat.compress"
#define MAPRED_OUTPUT_COMPRESSION_CODEC "mapreduce.output.fileoutputformat.compress.codec"
#define TOTAL_ORDER_PARTITIONER_PATH "total.order.partitioner.path"
@@ -386,32 +378,6 @@ public:
virtual bool next(Buffer & key, Buffer & value) = 0;
};
-class RecordReader : public KVIterator, public Configurable, public Progress {
-public:
- virtual NativeObjectType type() {
- return RecordReaderType;
- }
-
- virtual bool next(Buffer & key, Buffer & value) = 0;
-
- virtual float getProgress() = 0;
-
- virtual void close() = 0;
-};
-
-class RecordWriter : public Collector, public Configurable {
-public:
- virtual NativeObjectType type() {
- return RecordWriterType;
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) {
- }
-
- virtual void close() {
- }
-
-};
class ProcessorBase : public Configurable {
protected:
@@ -444,36 +410,6 @@ public:
}
};
-class Mapper : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return MapperType;
- }
-
- /**
- * Map interface, default IdenticalMapper
- */
- virtual void map(const char * key, uint32_t keyLen, const char * value, uint32_t valueLen) {
- collect(key, keyLen, value, valueLen);
- }
-};
-
-class Partitioner : public Configurable {
-public:
- virtual NativeObjectType type() {
- return PartitionerType;
- }
-
- /**
- * Partition interface
- * @param key key buffer
- * @param keyLen key length, can be modified to smaller value
- * to truncate key
- * @return partition number
- */
- virtual uint32_t getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition);
-};
-
enum KeyGroupIterState {
SAME_KEY,
NEW_KEY,
@@ -502,80 +438,7 @@ public:
virtual const char * nextValue(uint32_t & len) = 0;
};
-class Reducer : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return ReducerType;
- }
-
- /**
- * Reduce interface, default IdenticalReducer
- */
- virtual void reduce(KeyGroupIterator & input) {
- const char * key;
- const char * value;
- uint32_t keyLen;
- uint32_t valueLen;
- key = input.getKey(keyLen);
- while (NULL != (value = input.nextValue(valueLen))) {
- collect(key, keyLen, value, valueLen);
- }
- }
-};
-
-/**
- * Folder API used for hashtable based aggregation
- * Folder will be used in this way:
- * on(key, value):
- * state = hashtable.get(key)
- * if state == None:
- * size = size()
- * if size == -1:
- * state = init(null, -1)
- * elif size > 0:
- * state = fixallocator.get(key)
- * init(state, size)
- * folder(state, value, value.len)
- *
- * final():
- * for k,state in hashtable:
- * final(key, key.len, state)
- */
-class Folder : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return FolderType;
- }
-
- /**
- * Get aggregator state size
- * @return state storage size
- * -1 size not fixed or unknown, default
- * e.g. list map tree
- * 0 don't need to store state
- * >0 fixed sized state
- * e.g. int32 int64 float.
- */
- virtual int32_t size() {
- return -1;
- }
- /**
- * Create and/or init new state
- */
- virtual void * init(const char * key, uint32_t keyLen) {
- return NULL;
- }
-
- /**
- * Aggregation function
- */
- virtual void folder(void * dest, const char * value, uint32_t valueLen) {
- }
-
- virtual void final(const char * key, uint32_t keyLen, void * dest) {
- }
-};
enum KeyValueType {
TextType = 0,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
index 59cd8fb..ce36239 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
#include "BlockCodec.h"
@@ -79,9 +79,8 @@ BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t buff
}
void BlockDecompressStream::init() {
- _tempBufferSize = maxCompressedLength(_blockMax) + 8;
+ _tempBufferSize = maxCompressedLength(_blockMax) + 8;
_tempBuffer = (char*)malloc(_tempBufferSize);
-
}
BlockDecompressStream::~BlockDecompressStream() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
index 0412bb0..ffa4417 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
@@ -19,7 +19,7 @@
#ifndef BLOCKCODEC_H_
#define BLOCKCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
index bb699e9..b0b9852 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
@@ -18,7 +18,7 @@
#include <zconf.h>
#include <zlib.h>
-#include "commons.h"
+#include "lib/commons.h"
#include "GzipCodec.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
index 96721e5..a5f986e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
@@ -19,7 +19,7 @@
#ifndef GZIPCODEC_H_
#define GZIPCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
index 8dc4c64..75bfad1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lz4.h"
#include "NativeTask.h"
#include "Lz4Codec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
index d300dda..f3dd168 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
@@ -19,7 +19,7 @@
#ifndef LZ4CODEC_H_
#define LZ4CODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
#include "BlockCodec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
index 0504033..a0417e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
@@ -19,7 +19,7 @@
#include "config.h"
#if defined HADOOP_SNAPPY_LIBRARY
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
#include "SnappyCodec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
index 3ba06b8..0673cf3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
@@ -19,7 +19,7 @@
#ifndef SNAPPYCODEC_H_
#define SNAPPYCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
#include "BlockCodec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
index 7ed363b..37e0d42 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
@@ -16,20 +16,19 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "MCollectorOutputHandler.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
#include "CombineHandler.h"
using std::string;
using std::vector;
namespace NativeTask {
-
const Command AbstractMapHandler::GET_OUTPUT_PATH(100, "GET_OUTPUT_PATH");
const Command AbstractMapHandler::GET_OUTPUT_INDEX_PATH(101, "GET_OUTPUT_INDEX_PATH");
const Command AbstractMapHandler::GET_SPILL_PATH(102, "GET_SPILL_PATH");
const Command AbstractMapHandler::GET_COMBINE_HANDLER(103, "GET_COMBINE_HANDLER");
-} //namespace
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
index b5b7451..ecb217c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
@@ -19,11 +19,11 @@
#ifndef QUICK_BUILD
#include "org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h"
#endif
-#include "commons.h"
+#include "lib/commons.h"
#include "jni_md.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
#include "BatchHandler.h"
-#include "NativeObjectFactory.h"
+#include "lib/NativeObjectFactory.h"
///////////////////////////////////////////////////////////////
// NativeBatchProcessor jni util methods
@@ -130,7 +130,8 @@ void BatchHandler::onSetup(Config * config, char * inputBuffer, uint32_t inputBu
_out.reset(outputBuffer, outputBufferCapacity);
_out.rewind(0, outputBufferCapacity);
- LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d", inputBufferCapacity, _out.limit());
+ LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d",
+ inputBufferCapacity, _out.limit());
}
configure(_config);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
index 83da238..5f3863e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
@@ -208,7 +208,6 @@ void CombineHandler::write(char * buf, uint32_t length) {
outputRecordCount++;
remain -= kv->length();
pos += kv->length();
- ;
}
_combineOutputRecordCount += outputRecordCount;
@@ -242,7 +241,8 @@ void CombineHandler::combine(CombineContext type, KVIterator * kvIterator, IFile
this->_writer = writer;
call(COMBINE, NULL);
- LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, output Record Count: %d, output Bytes: %d",
+ LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, "
+ "output Record Count: %d, output Bytes: %d",
_combineInputRecordCount, _combineInputBytes,
_combineOutputRecordCount, _combineOutputBytes);
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
index 7f10552..eb43bd4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
@@ -18,7 +18,7 @@
#ifndef _COMBINEHANDLER_H_
#define _COMBINEHANDLER_H_
-#include "Combiner.h"
+#include "lib/Combiner.h"
#include "BatchHandler.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
index 4df5e64..7e4ae44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "lib/TaskCounters.h"
#include "MCollectorOutputHandler.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
#include "CombineHandler.h"
using std::string;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
index 33a1de9..e459b46 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
@@ -16,123 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "BufferStream.h"
+#include "lib/commons.h"
+#include "lib/BufferStream.h"
namespace NativeTask {
-BufferedInputStream::BufferedInputStream(InputStream * stream, uint32_t bufferSize)
- : FilterInputStream(stream), _buff(NULL), _position(0), _limit(0), _capacity(0) {
- _buff = (char*)malloc(bufferSize);
- if (NULL != _buff) {
- LOG("[BuferStream] malloc failed when create BufferedInputStream with buffersize %u",
- bufferSize);
- _capacity = bufferSize;
- }
-}
-
-BufferedInputStream::~BufferedInputStream() {
- if (NULL != _buff) {
- free(_buff);
- _buff = NULL;
- _position = 0;
- _limit = 0;
- _capacity = 0;
- }
-}
-
-void BufferedInputStream::seek(uint64_t position) {
- if (_limit - _position > 0) {
- THROW_EXCEPTION(IOException, "temporary buffered data exists when fseek()");
- }
- _stream->seek(position);
-}
-
-uint64_t BufferedInputStream::tell() {
- return _stream->tell() - (_limit - _position);
-}
-
-int32_t BufferedInputStream::read(void * buff, uint32_t length) {
- uint32_t rest = _limit - _position;
- if (rest > 0) {
- // have some data in buffer, read from buffer
- uint32_t cp = rest < length ? rest : length;
- memcpy(buff, _buff + _position, cp);
- _position += cp;
- return cp;
- } else if (length >= _capacity / 2) {
- // dest buffer big enough, read to dest buffer directly
- return _stream->read(buff, length);
- } else {
- // read to buffer first, then copy part of it to dest
- _limit = 0;
- do {
- int32_t rd = _stream->read(_buff + _limit, _capacity - _limit);
- if (rd <= 0) {
- break;
- }
- } while (_limit < _capacity / 2);
- if (_limit == 0) {
- return -1;
- }
- uint32_t cp = _limit < length ? _limit : length;
- memcpy(buff, _buff, cp);
- _position = cp;
- return cp;
- }
-}
-
-/////////////////////////////////////////////////////////////////
-
-BufferedOutputStream::BufferedOutputStream(InputStream * stream, uint32_t bufferSize)
- : FilterOutputStream(_stream), _buff(NULL), _position(0), _capacity(0) {
- _buff = (char*)malloc(bufferSize + sizeof(uint64_t));
- if (NULL != _buff) {
- LOG("[BuferStream] malloc failed when create BufferedOutputStream with buffersize %u",
- bufferSize);
- _capacity = bufferSize;
- }
-}
-
-BufferedOutputStream::~BufferedOutputStream() {
- if (NULL != _buff) {
- free(_buff);
- _buff = NULL;
- _position = 0;
- _capacity = 0;
- }
-}
-
-uint64_t BufferedOutputStream::tell() {
- return _stream->tell() + _position;
-}
-
-void BufferedOutputStream::write(const void * buff, uint32_t length) {
- if (length < _capacity / 2) {
- uint32_t rest = _capacity - _position;
- if (length < rest) {
- simple_memcpy(_buff + _position, buff, length);
- _position += length;
- } else {
- flush();
- simple_memcpy(_buff, buff, length);
- _position = length;
- }
- } else {
- flush();
- _stream->write(buff, length);
- }
-}
-
-void BufferedOutputStream::flush() {
- if (_position > 0) {
- _stream->write(_buff, _position);
- _position = 0;
- }
-}
-
-///////////////////////////////////////////////////////////
-
int32_t InputBuffer::read(void * buff, uint32_t length) {
uint32_t rd = _capacity - _position < length ? _capacity - _position : length;
if (rd > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
index 541bab5..e945b95 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
@@ -20,49 +20,12 @@
#define BUFFERSTREAM_H_
#include <string>
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
using std::string;
-class BufferedInputStream : public FilterInputStream {
-protected:
- char * _buff;
- uint32_t _position;
- uint32_t _limit;
- uint32_t _capacity;
-public:
- BufferedInputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024);
-
- virtual ~BufferedInputStream();
-
- virtual void seek(uint64_t position);
-
- virtual uint64_t tell();
-
- virtual int32_t read(void * buff, uint32_t length);
-};
-
-class BufferedOutputStream : public FilterOutputStream {
-protected:
- char * _buff;
- uint32_t _position;
- uint32_t _capacity;
-
-public:
- BufferedOutputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024);
-
- virtual ~BufferedOutputStream();
-
- virtual uint64_t tell();
-
- virtual void write(const void * buff, uint32_t length);
-
- virtual void flush();
-
-};
-
class InputBuffer : public InputStream {
protected:
const char * _buff;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
index d07f2fb..d40abb9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
@@ -16,84 +16,15 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <string>
+
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "util/WritableUtils.h"
-#include "Buffers.h"
+#include "lib/Buffers.h"
namespace NativeTask {
-DynamicBuffer::DynamicBuffer()
- : _data(NULL), _capacity(0), _size(0), _used(0) {
-}
-
-DynamicBuffer::DynamicBuffer(uint32_t capacity)
- : _data(NULL), _capacity(0), _size(0), _used(0) {
- reserve(capacity);
-}
-
-DynamicBuffer::~DynamicBuffer() {
- release();
-}
-
-void DynamicBuffer::release() {
- if (_data != NULL) {
- free(_data);
- _data = NULL;
- _capacity = 0;
- _used = 0;
- }
-}
-
-void DynamicBuffer::reserve(uint32_t capacity) {
- if (_data != NULL) {
- if (capacity > _capacity) {
- char * newdata = (char*)realloc(_data, capacity);
- if (newdata == NULL) {
- THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve realloc %u failed",
- capacity);
- }
- _data = newdata;
- _capacity = capacity;
- }
- return;
- }
- release();
- char * newdata = (char*)malloc(capacity);
- if (newdata == NULL) {
- THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve new %u failed", capacity);
- }
- _data = newdata;
- _capacity = capacity;
- _size = 0;
- _used = 0;
-}
-
-int32_t DynamicBuffer::refill(InputStream * stream) {
- if (_data == NULL || freeSpace() == 0) {
- THROW_EXCEPTION(IOException, "refill DynamicBuffer failed, no space left");
- }
- int32_t rd = stream->read(_data + _size, freeSpace());
- if (rd > 0) {
- _size += rd;
- }
- return rd;
-}
-
-void DynamicBuffer::cleanUsed() {
- if (_used > 0) {
- uint32_t needToMove = _size - _used;
- if (needToMove > 0) {
- memmove(_data, _data + _used, needToMove);
- _size = needToMove;
- } else {
- _size = 0;
- }
- _used = 0;
- }
-}
-
-///////////////////////////////////////////////////////////
ReadBuffer::ReadBuffer()
: _buff(NULL), _remain(0), _size(0), _capacity(0), _stream(NULL), _source(NULL) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
index 0da23af..13cd545 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
@@ -19,69 +19,12 @@
#ifndef BUFFERS_H_
#define BUFFERS_H_
-#include "Streams.h"
-#include "Compressions.h"
-#include "Constants.h"
+#include "lib/Streams.h"
+#include "lib/Compressions.h"
+#include "lib/Constants.h"
namespace NativeTask {
-class DynamicBuffer {
-protected:
- char * _data;
- uint32_t _capacity;
- uint32_t _size;
- uint32_t _used;
-public:
- DynamicBuffer();
-
- DynamicBuffer(uint32_t capacity);
-
- ~DynamicBuffer();
-
- void reserve(uint32_t capacity);
-
- void release();
-
- uint32_t capacity() {
- return _capacity;
- }
-
- char * data() {
- return _data;
- }
-
- uint32_t size() {
- return _size;
- }
-
- uint32_t used() {
- return _used;
- }
-
- char * current() {
- return _data + _used;
- }
-
- char * end() {
- return _data + _size;
- }
-
- uint32_t remain() {
- return _size - _used;
- }
-
- uint32_t freeSpace() {
- return _capacity - _size;
- }
-
- void use(uint32_t count) {
- _used += count;
- }
-
- void cleanUsed();
-
- int32_t refill(InputStream * stream);
-};
/**
* A lightweight read buffer, act as buffered input stream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
deleted file mode 100644
index b426fc3..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "Combiner.h"
-#include "StringUtil.h"
-
-namespace NativeTask {
-
-NativeCombineRunner::NativeCombineRunner(Config * config, ObjectCreatorFunc combinerCreator)
- : _config(config), _combinerCreator(combinerCreator), _keyGroupCount(0) {
- if (NULL == _combinerCreator) {
- THROW_EXCEPTION_EX(UnsupportException, "Create combiner failed");
- }
-}
-
-KeyGroupIterator * NativeCombineRunner::createKeyGroupIterator(KVIterator * iter) {
- return new KeyGroupIteratorImpl(iter);
-}
-
-void NativeCombineRunner::combine(CombineContext context, KVIterator * iterator,
- IFileWriter * writer) {
- Configurable * combiner = (Configurable *)(_combinerCreator());
- if (NULL != combiner) {
- combiner->configure(_config);
- }
-
- NativeObjectType type = combiner->type();
- switch (type) {
- case MapperType: {
- Mapper * mapper = (Mapper*)combiner;
- mapper->setCollector(writer);
-
- Buffer key;
- Buffer value;
- while (iterator->next(key, value)) {
- mapper->map(key.data(), key.length(), value.data(), value.length());
- }
- mapper->close();
- delete mapper;
- }
- break;
- case ReducerType: {
- Reducer * reducer = (Reducer*)combiner;
- reducer->setCollector(writer);
- KeyGroupIterator * kg = createKeyGroupIterator(iterator);
- while (kg->nextKey()) {
- _keyGroupCount++;
- reducer->reduce(*kg);
- }
- reducer->close();
- delete reducer;
- }
- break;
- default:
- THROW_EXCEPTION(UnsupportException, "Combiner type not support");
- }
-}
-
-} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
index 09f4eb0..a7309ff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
@@ -18,7 +18,7 @@
#ifndef COMBINER_H_
#define COMBINER_H_
#include "commons.h"
-#include "IFile.h"
+#include "lib/IFile.h"
namespace NativeTask {
@@ -66,21 +66,5 @@ public:
}
};
-class NativeCombineRunner : public ICombineRunner {
-private:
- Config * _config;
- ObjectCreatorFunc _combinerCreator;
- uint32_t _keyGroupCount;
-
-public:
- NativeCombineRunner(Config * config, ObjectCreatorFunc objectCreator);
-
-public:
- void combine(CombineContext type, KVIterator * kvIterator, IFileWriter * writer);
-
-private:
- KeyGroupIterator * createKeyGroupIterator(KVIterator * iter);
-};
-
} /* namespace NativeTask */
#endif /* COMBINER_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
index 0f954e0..879c016 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "config.h"
-#include "SyncUtils.h"
-#include "Compressions.h"
+#include "lib/Compressions.h"
+#include "util/SyncUtils.h"
#include "codec/GzipCodec.h"
#include "codec/SnappyCodec.h"
#include "codec/Lz4Codec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
index f70adb6..96c5084 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
@@ -21,7 +21,7 @@
#include <string>
#include <vector>
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
index e137883..e0a698c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
@@ -20,14 +20,14 @@
#include <fcntl.h>
#include <dirent.h>
#include <sys/stat.h>
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
#include "NativeTask.h"
-#include "TaskCounters.h"
-#include "NativeObjectFactory.h"
-#include "Path.h"
-#include "FileSystem.h"
+#include "lib/TaskCounters.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/Path.h"
+#include "lib/FileSystem.h"
namespace NativeTask {
@@ -122,14 +122,15 @@ void FileOutputStream::close() {
/////////////////////////////////////////////////////////////
class RawFileSystem : public FileSystem {
-protected:
+ protected:
string getRealPath(const string & path) {
if (StringUtil::StartsWith(path, "file:")) {
return path.substr(5);
}
return path;
}
-public:
+
+ public:
InputStream * open(const string & path) {
return new FileInputStream(getRealPath(path));
}
@@ -206,7 +207,7 @@ public:
struct stat sb;
if (stat(np.c_str(), &sb) == 0) {
- if (S_ISDIR (sb.st_mode) == 0) {
+ if (S_ISDIR(sb.st_mode) == 0) {
return 1;
}
return 0;
@@ -226,7 +227,7 @@ public:
if (mkdir(npath, nmode)) {
return 1;
}
- } else if (S_ISDIR (sb.st_mode) == 0) {
+ } else if (S_ISDIR(sb.st_mode) == 0) {
return 1;
}
*p++ = '/'; /* restore slash */
@@ -259,4 +260,4 @@ FileSystem & FileSystem::getLocal() {
return RawFileSystemInstance;
}
-} // namespace Hadoap
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
index 7649947..8eaf32d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
@@ -21,7 +21,7 @@
#include <string>
#include "NativeTask.h"
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
index 4a5edda..2d3e0b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
-#include "IFile.h"
-#include "Compressions.h"
+#include "lib/IFile.h"
+#include "lib/Compressions.h"
#include "lib/FileSystem.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
index d86e98a..e397f90 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
@@ -19,11 +19,11 @@
#ifndef IFILE_H_
#define IFILE_H_
-#include "Checksum.h"
-#include "Buffers.h"
-#include "WritableUtils.h"
-#include "SpillInfo.h"
-#include "MapOutputSpec.h"
+#include "util/Checksum.h"
+#include "lib/Buffers.h"
+#include "util/WritableUtils.h"
+#include "lib/SpillInfo.h"
+#include "lib/MapOutputSpec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
index dfde456..3d0516a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
@@ -15,8 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#include "Iterator.h"
-#include "commons.h"
+#include "lib/Iterator.h"
+#include "lib/commons.h"
namespace NativeTask {
@@ -88,4 +88,4 @@ bool KeyGroupIteratorImpl::next() {
return result;
}
-} //namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
index 2eaa342..1e36e18 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "Log.h"
+#include "lib/Log.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
index d151f1f..1175b23 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
@@ -16,19 +16,21 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <string>
+
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "FileSystem.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
-#include "Merge.h"
+#include "lib/FileSystem.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
@@ -36,22 +38,18 @@ ICombineRunner * CombineRunnerWrapper::createCombiner() {
ICombineRunner * combineRunner = NULL;
if (NULL != _config->get(NATIVE_COMBINER)) {
- const char * combinerClass = _config->get(NATIVE_COMBINER);
- ObjectCreatorFunc objectCreater = NativeObjectFactory::GetObjectCreator(combinerClass);
- if (NULL == objectCreater) {
- THROW_EXCEPTION_EX(UnsupportException, "Combiner not found: %s", combinerClass);
- } else {
- LOG("[MapOutputCollector::configure] native combiner is enabled: %s", combinerClass);
- }
- combineRunner = new NativeCombineRunner(_config, objectCreater);
+ // Earlier versions of this code supported user-defined
+ // native Combiner implementations. This simplified version
+ // no longer supports it.
+ THROW_EXCEPTION_EX(UnsupportException, "Native Combiners not supported");
+ }
+
+ CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler();
+ if (NULL != javaCombiner) {
+ _isJavaCombiner = true;
+ combineRunner = (ICombineRunner *)javaCombiner;
} else {
- CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler();
- if (NULL != javaCombiner) {
- _isJavaCombiner = true;
- combineRunner = (ICombineRunner *)javaCombiner;
- } else {
- LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java");
- }
+ LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java");
}
return combineRunner;
}
@@ -118,7 +116,7 @@ void MapOutputCollector::init(uint32_t defaultBlockSize, uint32_t memoryCapacity
_pool->init(memoryCapacity);
- //TODO: add support for customized comparator
+ // TODO: add support for customized comparator
this->_keyComparator = keyComparator;
_buckets = new PartitionBucket*[_numPartitions];
@@ -160,7 +158,8 @@ void MapOutputCollector::configure(Config * config) {
uint32_t capacity = config->getInt(MAPRED_IO_SORT_MB, 300) * 1024 * 1024;
uint32_t defaultBlockSize = getDefaultBlockSize(capacity, _numPartitions, maxBlockSize);
- LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024,
+ LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, "
+ "max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024,
maxBlockSize / 1024, capacity / 1024 / 1024);
ComparatorPtr comparator = getComparator(config, _spec);
@@ -179,7 +178,8 @@ void MapOutputCollector::configure(Config * config) {
KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t kvlength) {
PartitionBucket * partition = getPartition(partitionId);
if (NULL == partition) {
- THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d", partitionId, _numPartitions);
+ THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d",
+ partitionId, _numPartitions);
}
KVBuffer * dest = partition->allocateKVBuffer(kvlength);
@@ -301,7 +301,7 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
info->path = spillOutput;
uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime;
- const uint64_t M = 1000000; //million
+ const uint64_t M = 1000000; // million
LOG("%s-spill: { id: %d, collect: %"PRIu64" ms, "
"in-memory sort: %"PRIu64" ms, in-memory records: %"PRIu64", "
"merge&spill: %"PRIu64" ms, uncompressed size: %"PRIu64", "
@@ -369,7 +369,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
uint64_t recordCount;
writer->getStatistics(outputSize, realOutputSize, recordCount);
- const uint64_t M = 1000000; //million
+ const uint64_t M = 1000000; // million
LOG("Final-merge-spill: { id: %d, in-memory sort: %"PRIu64" ms, "
"in-memory records: %"PRIu64", merge&spill: %"PRIu64" ms, "
"records: %"PRIu64", uncompressed size: %"PRIu64", "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
index 474b8b1..ee447ed 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
@@ -20,14 +20,14 @@
#define MAP_OUTPUT_COLLECTOR_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "PartitionBucket.h"
+#include "lib/MemoryPool.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
+#include "lib/PartitionBucket.h"
#include "lib/SpillOutputService.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
index 5088206..8874d02 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
@@ -16,8 +16,8 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "MapOutputSpec.h"
+#include "lib/commons.h"
+#include "lib/MapOutputSpec.h"
#include "NativeTask.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
index fac96ab..ce43997 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
@@ -20,8 +20,8 @@
#define MAPOUTPUTSPEC_H_
#include <string>
-#include "Checksum.h"
-#include "WritableUtils.h"
+#include "util/Checksum.h"
+#include "util/WritableUtils.h"
#include "NativeTask.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
index 29f59b9..b3734a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
@@ -15,18 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+#include <algorithm>
+
#include "NativeTask.h"
-#include "commons.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "MapOutputSpec.h"
+#include "lib/commons.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
-#include "MemoryBlock.h"
-#include "MemoryPool.h"
+#include "lib/MemoryBlock.h"
+#include "lib/MemoryPool.h"
#include "util/DualPivotQuickSort.h"
namespace NativeTask {
@@ -62,4 +64,4 @@ void MemoryBlock::sort(SortAlgorithm type, ComparatorPtr comparator) {
}
_sorted = true;
}
-} //namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
index 0303fb2..1aa9cc2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
@@ -19,8 +19,8 @@
#ifndef MEMORYPOOL_H_
#define MEMORYPOOL_H_
-#include "Buffers.h"
-#include "MapOutputSpec.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
#include "NativeTask.h"
#include "util/StringUtil.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
index 126b82d..0d18db8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "Merge.h"
+#include "lib/Merge.h"
#include "lib/FileSystem.h"
namespace NativeTask {
@@ -34,7 +34,6 @@ Merger::Merger(IFileWriter * writer, Config * config, ComparatorPtr comparator,
ICombineRunner * combineRunner)
: _writer(writer), _config(config), _combineRunner(combineRunner), _first(true),
_comparator(comparator) {
-
}
Merger::~Merger() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
index 956fa41..cf1f2be 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
@@ -20,10 +20,10 @@
#define MERGE_H_
#include "NativeTask.h"
-#include "Buffers.h"
-#include "MapOutputCollector.h"
-#include "IFile.h"
-#include "MinHeap.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputCollector.h"
+#include "lib/IFile.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
index ab434f6..5885f34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
@@ -20,7 +20,7 @@
#define MIN_HEAP_H_
#include "NativeTask.h"
-#include "Buffers.h"
+#include "lib/Buffers.h"
template<typename T, typename Compare>
void heapify(T* first, int rt, int heap_len, Compare & Comp) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
index 237b55e..5d69b26 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
@@ -18,9 +18,9 @@
#include <dlfcn.h>
-#include "commons.h"
-#include "NativeObjectFactory.h"
-#include "NativeLibrary.h"
+#include "lib/commons.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/NativeLibrary.h"
namespace NativeTask {
@@ -30,7 +30,6 @@ namespace NativeTask {
NativeLibrary::NativeLibrary(const string & path, const string & name)
: _path(path), _name(name), _getObjectCreatorFunc(NULL), _functionGetter(NULL) {
-
}
bool NativeLibrary::init() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
index ce3799d..2185798 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
@@ -20,11 +20,11 @@
#ifndef __CYGWIN__
#include <execinfo.h>
#endif
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
-#include "NativeObjectFactory.h"
-#include "NativeLibrary.h"
-#include "BufferStream.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/NativeLibrary.h"
+#include "lib/BufferStream.h"
#include "util/StringUtil.h"
#include "util/SyncUtils.h"
#include "util/WritableUtils.h"
@@ -53,19 +53,10 @@ extern "C" void handler(int sig) {
}
DEFINE_NATIVE_LIBRARY(NativeTask) {
- //signal(SIGSEGV, handler);
REGISTER_CLASS(BatchHandler, NativeTask);
REGISTER_CLASS(CombineHandler, NativeTask);
REGISTER_CLASS(MCollectorOutputHandler, NativeTask);
- REGISTER_CLASS(Mapper, NativeTask);
- REGISTER_CLASS(Reducer, NativeTask);
- REGISTER_CLASS(Partitioner, NativeTask);
- REGISTER_CLASS(Folder, NativeTask);
NativeObjectFactory::SetDefaultClass(BatchHandlerType, "NativeTask.BatchHandler");
- NativeObjectFactory::SetDefaultClass(MapperType, "NativeTask.Mapper");
- NativeObjectFactory::SetDefaultClass(ReducerType, "NativeTask.Reducer");
- NativeObjectFactory::SetDefaultClass(PartitionerType, "NativeTask.Partitioner");
- NativeObjectFactory::SetDefaultClass(FolderType, "NativeTask.Folder");
}
namespace NativeTask {
@@ -132,7 +123,7 @@ bool NativeObjectFactory::Init() {
}
const char * version = GetConfig().get(NATIVE_HADOOP_VERSION);
LOG("[NativeObjectLibrary] NativeTask library initialized with hadoop %s",
- version==NULL?"unkown":version);
+ version == NULL ? "unkown" : version);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
index 28aeaf3..be682e8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
@@ -20,9 +20,9 @@
#include "org_apache_hadoop_mapred_nativetask_NativeRuntime.h"
#endif
#include "config.h"
-#include "commons.h"
-#include "jniutils.h"
-#include "NativeObjectFactory.h"
+#include "lib/commons.h"
+#include "lib/jniutils.h"
+#include "lib/NativeObjectFactory.h"
using namespace NativeTask;
@@ -36,8 +36,7 @@ using namespace NativeTask;
* Signature: ([B)Z
*/
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
- (JNIEnv *jenv, jclass clazz, jbyteArray codec)
-{
+ (JNIEnv *jenv, jclass clazz, jbyteArray codec) {
const std::string codecString = JNU_ByteArrayToString(jenv, codec);
if ("org.apache.hadoop.io.compress.GzipCodec" == codecString) {
return JNI_TRUE;
@@ -58,7 +57,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
JNIEnv * jenv, jclass nativeRuntimeClass) {
try {
NativeTask::NativeObjectFactory::Release();
@@ -82,7 +82,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIConfigure
* Signature: ([[B)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure(
JNIEnv * jenv, jclass nativeRuntimeClass, jobjectArray configs) {
try {
NativeTask::Config & config = NativeTask::NativeObjectFactory::GetConfig();
@@ -139,7 +140,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNICreateDefaultNativeObject
* Signature: ([B)J
- */JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject(
+ */
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject(
JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray type) {
try {
std::string typeString = JNU_ByteArrayToString(jenv, type);
@@ -166,7 +168,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIReleaseNativeObject
* Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject(
JNIEnv * jenv, jclass nativeRuntimeClass, jlong objectAddr) {
try {
NativeTask::NativeObject * nobj = ((NativeTask::NativeObject *)objectAddr);
@@ -196,7 +199,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRegisterModule
* Signature: ([B[B)I
- */JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule(
+ */
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule(
JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray modulePath, jbyteArray moduleName) {
try {
std::string pathString = JNU_ByteArrayToString(jenv, modulePath);
@@ -225,7 +229,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIUpdateStatus
* Signature: ()[B
- */JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus(
+ */
+JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus(
JNIEnv * jenv, jclass nativeRuntimeClass) {
try {
std::string statusData;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
index 5c42e56..5dc880c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
@@ -18,11 +18,10 @@
#ifndef __CYGWIN__
#include <execinfo.h>
#endif
-#include "commons.h"
-#include "util/Hash.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "NativeTask.h"
-#include "NativeObjectFactory.h"
+#include "lib/NativeObjectFactory.h"
namespace NativeTask {
@@ -34,20 +33,6 @@ const string NativeObjectTypeToString(NativeObjectType type) {
switch (type) {
case BatchHandlerType:
return string("BatchHandlerType");
- case MapperType:
- return string("MapperType");
- case ReducerType:
- return string("ReducerType");
- case PartitionerType:
- return string("PartitionerType");
- case CombinerType:
- return string("CombinerType");
- case FolderType:
- return string("FolderType");
- case RecordReaderType:
- return string("RecordReaderType");
- case RecordWriterType:
- return string("RecordWriterType");
default:
return string("UnknownObjectType");
}
@@ -56,20 +41,6 @@ const string NativeObjectTypeToString(NativeObjectType type) {
NativeObjectType NativeObjectTypeFromString(const string type) {
if (type == "BatchHandlerType") {
return BatchHandlerType;
- } else if (type == "MapperType") {
- return MapperType;
- } else if (type == "ReducerType") {
- return ReducerType;
- } else if (type == "PartitionerType") {
- return PartitionerType;
- } else if (type == "CombinerType") {
- return CombinerType;
- } else if (type == "FolderType") {
- return CombinerType;
- } else if (type == "RecordReaderType") {
- return RecordReaderType;
- } else if (type == "RecordWriterType") {
- return RecordWriterType;
}
return UnknownObjectType;
}
@@ -96,7 +67,7 @@ HadoopException::HadoopException(const string & what) {
#ifndef __CYGWIN__
size = backtrace(array, 64);
char ** traces = backtrace_symbols(array, size);
- for (size_t i = 0; i< size;i++) {
+ for (size_t i = 0; i < size; i++) {
_reason.append("\n\t");
_reason.append(traces[i]);
}
@@ -235,13 +206,6 @@ Counter * ProcessorBase::getCounter(const string & group, const string & name) {
return NULL;
}
-uint32_t Partitioner::getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition) {
- if (numPartition == 1) {
- return 0;
- }
- return (Hash::BytesHash(key, keyLen) & 0x7fffffff) % numPartition;
-}
-
///////////////////////////////////////////////////////////
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
index 55ed4b2..33c2403 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
@@ -16,19 +16,19 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "NativeObjectFactory.h"
-#include "PartitionBucket.h"
-#include "Merge.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/PartitionBucket.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
-#include "PartitionBucketIterator.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
+#include "lib/PartitionBucketIterator.h"
namespace NativeTask {
@@ -39,7 +39,8 @@ KVIterator * PartitionBucket::getIterator() {
return new PartitionBucketIterator(this, _keyComparator);
}
-void PartitionBucket::spill(IFileWriter * writer) throw (IOException, UnsupportException) {
+void PartitionBucket::spill(IFileWriter * writer)
+ throw(IOException, UnsupportException) {
KVIterator * iterator = getIterator();
if (NULL == iterator || NULL == writer) {
return;
@@ -71,6 +72,4 @@ void PartitionBucket::sort(SortAlgorithm type) {
_sorted = true;
}
-}
-;
-// namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
index 3be52ff..cd9ec97 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
@@ -20,14 +20,14 @@
#define PARTITION_BUCKET_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "MemoryBlock.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
+#include "lib/MemoryPool.h"
+#include "lib/MemoryBlock.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
index 0b26f21..ade042e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
@@ -16,18 +16,20 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <algorithm>
+
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "NativeObjectFactory.h"
-#include "PartitionBucketIterator.h"
-#include "Merge.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/PartitionBucketIterator.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
@@ -108,7 +110,5 @@ bool PartitionBucketIterator::next(Buffer & key, Buffer & value) {
}
return false;
}
-}
-;
-// namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
index beabff8..be78497 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
@@ -20,14 +20,14 @@
#define PARTITION_BUCKET_ITERATOR_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "PartitionBucket.h"
+#include "lib/MemoryPool.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
+#include "lib/PartitionBucket.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
index 6d85fe2..78321e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "Path.h"
+#include "lib/Path.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
index bce2cb2..9cff529 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "Streams.h"
-#include "FileSystem.h"
-#include "Buffers.h"
-#include "SpillInfo.h"
+#include "lib/commons.h"
+#include "lib/Streams.h"
+#include "lib/FileSystem.h"
+#include "lib/Buffers.h"
+#include "lib/SpillInfo.h"
namespace NativeTask {
@@ -69,5 +69,5 @@ void SingleSpillInfo::writeSpillInfo(const std::string & filepath) {
delete fout;
}
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
index c2d64c9..9e8ffba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "Checksum.h"
-#include "Streams.h"
+#include "lib/commons.h"
+#include "util/Checksum.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
index 39a7c43..b70db4c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "TaskCounters.h"
+#include "lib/TaskCounters.h"
namespace NativeTask {
@@ -36,6 +36,5 @@ const char * TaskCounters::FILESYSTEM_COUNTER_GROUP = "FileSystemCounters";
DEFINE_COUNTER(FILE_BYTES_READ)
DEFINE_COUNTER(FILE_BYTES_WRITTEN)
-;
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
deleted file mode 100644
index 3f96faf..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef TRACKING_COLLECTOR_H
-#define TRACKING_COLLECTOR_H
-
-#include <stdint.h>
-#include <string>
-
-namespace NativeTask {
-
-class TrackingCollector : public Collector {
-protected:
- Collector * _collector;
- Counter * _counter;
-public:
- TrackingCollector(Collector * collector, Counter * counter)
- : _collector(collector), _counter(counter) {
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) {
- _counter->increase();
- _collector->collect(key, keyLen, value, valueLen);
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen,
- int32_t partition) {
- _counter->increase();
- _collector->collect(key, keyLen, value, valueLen, partition);
- }
-};
-
-} //namespace NativeTask
-
-#endif //TRACKING_COLLECTOR_H
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
index 4212533..57500b7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
@@ -41,13 +41,12 @@
#include <map>
#include <algorithm>
-#include "primitives.h"
-#include "Log.h"
+#include "lib/primitives.h"
+#include "lib/Log.h"
#include "NativeTask.h"
-#include "Constants.h"
+#include "lib/Constants.h"
-#include "Iterator.h"
-#include "TrackingCollector.h"
+#include "lib/Iterator.h"
#endif /* COMMONS_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
index 01eaa57..d1c45bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "util/SyncUtils.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
using namespace NativeTask;
@@ -53,7 +53,7 @@ JavaVM * JNU_GetJVM(void) {
JavaVMOption options[noArgs];
options[0].optionString = optHadoopClassPath;
- //Create the VM
+ // Create the VM
JavaVMInitArgs vm_args;
vm_args.version = JNI_VERSION_1_6;
vm_args.options = options;
@@ -108,4 +108,4 @@ std::string JNU_ByteArrayToString(JNIEnv * jenv, jbyteArray src) {
return ret;
}
return std::string();
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
index 4c4f3cc..191e093 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
@@ -17,7 +17,7 @@
*/
#include <assert.h>
-#include "Checksum.h"
+#include "util/Checksum.h"
namespace NativeTask {
[48/50] [abbrv] git commit: MAPREDUCE-2841. Merge remote-tracking
branch 'apache/MR-2841' into trunk
Posted by ar...@apache.org.
MAPREDUCE-2841. Merge remote-tracking branch 'apache/MR-2841' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a3a327c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a3a327c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a3a327c
Branch: refs/heads/HDFS-6581
Commit: 8a3a327cd3c7fd1049b521325e485c59137855e8
Parents: a0ad975 b160707
Author: Todd Lipcon <to...@apache.org>
Authored: Fri Sep 12 18:31:53 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 12 18:31:53 2014 -0700
----------------------------------------------------------------------
LICENSE.txt | 33 +
.../assemblies/hadoop-mapreduce-dist.xml | 7 +
.../CHANGES.MAPREDUCE-2841.txt | 26 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 216 +
.../src/CMakeLists.txt | 276 +
.../src/JNIFlags.cmake | 118 +
.../src/config.h.cmake | 23 +
.../hadoop/mapred/nativetask/Command.java | 57 +
.../mapred/nativetask/CommandDispatcher.java | 33 +
.../hadoop/mapred/nativetask/Constants.java | 48 +
.../hadoop/mapred/nativetask/DataChannel.java | 41 +
.../hadoop/mapred/nativetask/DataReceiver.java | 37 +
.../mapred/nativetask/HadoopPlatform.java | 84 +
.../mapred/nativetask/ICombineHandler.java | 44 +
.../mapred/nativetask/INativeComparable.java | 54 +
.../mapred/nativetask/INativeHandler.java | 53 +
.../mapred/nativetask/NativeBatchProcessor.java | 279 +
.../mapred/nativetask/NativeDataSource.java | 47 +
.../mapred/nativetask/NativeDataTarget.java | 47 +
.../NativeMapOutputCollectorDelegator.java | 171 +
.../hadoop/mapred/nativetask/NativeRuntime.java | 197 +
.../hadoop/mapred/nativetask/Platform.java | 100 +
.../hadoop/mapred/nativetask/Platforms.java | 79 +
.../mapred/nativetask/StatusReportChecker.java | 99 +
.../hadoop/mapred/nativetask/TaskContext.java | 94 +
.../mapred/nativetask/buffer/BufferType.java | 27 +
.../nativetask/buffer/ByteBufferDataReader.java | 148 +
.../nativetask/buffer/ByteBufferDataWriter.java | 169 +
.../nativetask/buffer/DataInputStream.java | 27 +
.../nativetask/buffer/DataOutputStream.java | 39 +
.../mapred/nativetask/buffer/InputBuffer.java | 136 +
.../mapred/nativetask/buffer/OutputBuffer.java | 76 +
.../nativetask/handlers/BufferPullee.java | 121 +
.../nativetask/handlers/BufferPuller.java | 203 +
.../nativetask/handlers/BufferPushee.java | 151 +
.../nativetask/handlers/BufferPusher.java | 91 +
.../nativetask/handlers/CombinerHandler.java | 145 +
.../mapred/nativetask/handlers/IDataLoader.java | 37 +
.../handlers/NativeCollectorOnlyHandler.java | 171 +
.../serde/BoolWritableSerializer.java | 35 +
.../serde/ByteWritableSerializer.java | 35 +
.../serde/BytesWritableSerializer.java | 48 +
.../nativetask/serde/DefaultSerializer.java | 71 +
.../serde/DoubleWritableSerializer.java | 35 +
.../serde/FloatWritableSerializer.java | 36 +
.../mapred/nativetask/serde/IKVSerializer.java | 48 +
.../nativetask/serde/INativeSerializer.java | 49 +
.../nativetask/serde/IntWritableSerializer.java | 35 +
.../mapred/nativetask/serde/KVSerializer.java | 115 +
.../serde/LongWritableSerializer.java | 34 +
.../nativetask/serde/NativeSerialization.java | 91 +
.../serde/NullWritableSerializer.java | 35 +
.../serde/SerializationFramework.java | 35 +
.../mapred/nativetask/serde/TextSerializer.java | 49 +
.../serde/VIntWritableSerializer.java | 27 +
.../serde/VLongWritableSerializer.java | 27 +
.../mapred/nativetask/util/BytesUtil.java | 172 +
.../mapred/nativetask/util/ConfigUtil.java | 47 +
.../nativetask/util/LocalJobOutputFiles.java | 159 +
.../nativetask/util/NativeTaskOutput.java | 106 +
.../nativetask/util/NativeTaskOutputFiles.java | 170 +
.../mapred/nativetask/util/OutputUtil.java | 47 +
.../mapred/nativetask/util/ReadWriteBuffer.java | 159 +
.../mapred/nativetask/util/SizedWritable.java | 55 +
.../src/main/native/gtest/gtest-all.cc | 9120 ++++++++
.../src/main/native/gtest/gtest_main.cc | 39 +
.../src/main/native/gtest/include/gtest/gtest.h | 19541 +++++++++++++++++
.../src/main/native/src/NativeTask.h | 507 +
.../src/main/native/src/codec/BlockCodec.cc | 179 +
.../src/main/native/src/codec/BlockCodec.h | 98 +
.../src/main/native/src/codec/GzipCodec.cc | 192 +
.../src/main/native/src/codec/GzipCodec.h | 82 +
.../src/main/native/src/codec/Lz4Codec.cc | 88 +
.../src/main/native/src/codec/Lz4Codec.h | 45 +
.../src/main/native/src/codec/SnappyCodec.cc | 98 +
.../src/main/native/src/codec/SnappyCodec.h | 46 +
.../native/src/handler/AbstractMapHandler.cc | 34 +
.../native/src/handler/AbstractMapHandler.h | 102 +
.../src/main/native/src/handler/BatchHandler.cc | 355 +
.../src/main/native/src/handler/BatchHandler.h | 157 +
.../main/native/src/handler/CombineHandler.cc | 254 +
.../main/native/src/handler/CombineHandler.h | 90 +
.../src/handler/MCollectorOutputHandler.cc | 98 +
.../src/handler/MCollectorOutputHandler.h | 53 +
.../src/main/native/src/lib/BufferStream.cc | 43 +
.../src/main/native/src/lib/BufferStream.h | 156 +
.../src/main/native/src/lib/Buffers.cc | 235 +
.../src/main/native/src/lib/Buffers.h | 637 +
.../src/main/native/src/lib/Combiner.h | 70 +
.../src/main/native/src/lib/Compressions.cc | 145 +
.../src/main/native/src/lib/Compressions.h | 108 +
.../src/main/native/src/lib/Constants.h | 26 +
.../src/main/native/src/lib/FileSystem.cc | 263 +
.../src/main/native/src/lib/FileSystem.h | 123 +
.../src/main/native/src/lib/IFile.cc | 230 +
.../src/main/native/src/lib/IFile.h | 166 +
.../src/main/native/src/lib/Iterator.cc | 91 +
.../src/main/native/src/lib/Iterator.h | 47 +
.../src/main/native/src/lib/Log.cc | 30 +
.../src/main/native/src/lib/Log.h | 48 +
.../main/native/src/lib/MapOutputCollector.cc | 413 +
.../main/native/src/lib/MapOutputCollector.h | 167 +
.../src/main/native/src/lib/MapOutputSpec.cc | 64 +
.../src/main/native/src/lib/MapOutputSpec.h | 81 +
.../src/main/native/src/lib/MemoryBlock.cc | 67 +
.../src/main/native/src/lib/MemoryBlock.h | 172 +
.../src/main/native/src/lib/MemoryPool.h | 90 +
.../src/main/native/src/lib/Merge.cc | 156 +
.../src/main/native/src/lib/Merge.h | 254 +
.../src/main/native/src/lib/MinHeap.h | 64 +
.../src/main/native/src/lib/NativeLibrary.cc | 89 +
.../src/main/native/src/lib/NativeLibrary.h | 57 +
.../main/native/src/lib/NativeObjectFactory.cc | 444 +
.../main/native/src/lib/NativeObjectFactory.h | 106 +
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 256 +
.../src/main/native/src/lib/NativeTask.cc | 211 +
.../src/main/native/src/lib/PartitionBucket.cc | 75 +
.../src/main/native/src/lib/PartitionBucket.h | 142 +
.../native/src/lib/PartitionBucketIterator.cc | 114 +
.../native/src/lib/PartitionBucketIterator.h | 54 +
.../src/main/native/src/lib/Path.cc | 53 +
.../src/main/native/src/lib/Path.h | 38 +
.../src/main/native/src/lib/SpillInfo.cc | 73 +
.../src/main/native/src/lib/SpillInfo.h | 106 +
.../main/native/src/lib/SpillOutputService.h | 44 +
.../src/main/native/src/lib/Streams.cc | 122 +
.../src/main/native/src/lib/Streams.h | 221 +
.../src/main/native/src/lib/TaskCounters.cc | 40 +
.../src/main/native/src/lib/TaskCounters.h | 44 +
.../src/main/native/src/lib/commons.h | 52 +
.../src/main/native/src/lib/jniutils.cc | 111 +
.../src/main/native/src/lib/jniutils.h | 57 +
.../src/main/native/src/lib/primitives.h | 287 +
.../src/main/native/src/util/Checksum.cc | 749 +
.../src/main/native/src/util/Checksum.h | 78 +
.../main/native/src/util/DualPivotQuickSort.h | 137 +
.../src/main/native/src/util/Random.cc | 285 +
.../src/main/native/src/util/Random.h | 140 +
.../src/main/native/src/util/StringUtil.cc | 213 +
.../src/main/native/src/util/StringUtil.h | 66 +
.../src/main/native/src/util/SyncUtils.cc | 55 +
.../src/main/native/src/util/SyncUtils.h | 71 +
.../src/main/native/src/util/Timer.cc | 116 +
.../src/main/native/src/util/Timer.h | 56 +
.../src/main/native/src/util/WritableUtils.cc | 309 +
.../src/main/native/src/util/WritableUtils.h | 124 +
.../src/main/native/test.sh | 36 +
.../src/main/native/test/TestCommand.cc | 39 +
.../src/main/native/test/TestCompressions.cc | 286 +
.../src/main/native/test/TestConfig.cc | 67 +
.../src/main/native/test/TestCounter.cc | 48 +
.../src/main/native/test/TestFileSystem.cc | 50 +
.../src/main/native/test/TestIFile.cc | 199 +
.../src/main/native/test/TestMain.cc | 116 +
.../src/main/native/test/TestPrimitives.cc | 304 +
.../src/main/native/test/TestSort.cc | 306 +
.../src/main/native/test/lib/TestByteArray.cc | 43 +
.../src/main/native/test/lib/TestByteBuffer.cc | 47 +
.../lib/TestComparatorForDualPivotQuickSort.cc | 89 +
.../native/test/lib/TestComparatorForStdSort.cc | 89 +
.../native/test/lib/TestFixSizeContainer.cc | 55 +
.../src/main/native/test/lib/TestIterator.cc | 114 +
.../src/main/native/test/lib/TestKVBuffer.cc | 54 +
.../native/test/lib/TestMemBlockIterator.cc | 110 +
.../src/main/native/test/lib/TestMemoryBlock.cc | 109 +
.../src/main/native/test/lib/TestMemoryPool.cc | 48 +
.../main/native/test/lib/TestPartitionBucket.cc | 222 +
.../src/main/native/test/lib/TestReadBuffer.cc | 69 +
.../main/native/test/lib/TestReadWriteBuffer.cc | 70 +
.../src/main/native/test/test_commons.cc | 344 +
.../src/main/native/test/test_commons.h | 150 +
.../src/main/native/test/util/TestChecksum.cc | 43 +
.../src/main/native/test/util/TestProcess.cc | 33 +
.../src/main/native/test/util/TestStringUtil.cc | 64 +
.../main/native/test/util/TestWritableUtils.cc | 49 +
.../main/native/testData/testGlibCBugSpill.out | 2 +
...org.apache.hadoop.mapred.nativetask.Platform | 14 +
.../mapred/nativetask/TestTaskContext.java | 44 +
.../nativetask/buffer/TestBufferPushPull.java | 279 +
.../buffer/TestByteBufferReadWrite.java | 168 +
.../nativetask/buffer/TestInputBuffer.java | 53 +
.../nativetask/buffer/TestOutputBuffer.java | 45 +
.../nativetask/combinertest/CombinerTest.java | 124 +
.../combinertest/LargeKVCombinerTest.java | 115 +
.../combinertest/OldAPICombinerTest.java | 129 +
.../nativetask/combinertest/WordCount.java | 91 +
.../combinertest/WordCountWithOldAPI.java | 68 +
.../nativetask/compresstest/CompressMapper.java | 63 +
.../nativetask/compresstest/CompressTest.java | 144 +
.../nativetask/handlers/TestCombineHandler.java | 80 +
.../TestNativeCollectorOnlyHandler.java | 129 +
.../nativetask/kvtest/HashSumReducer.java | 49 +
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 108 +
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 153 +
.../mapred/nativetask/kvtest/LargeKVTest.java | 131 +
.../mapred/nativetask/kvtest/TestInputFile.java | 227 +
.../nativetask/nonsorttest/NonSortTest.java | 118 +
.../nativetask/nonsorttest/NonSortTestMR.java | 72 +
.../nativetask/serde/TestKVSerializer.java | 148 +
.../serde/TestNativeSerialization.java | 114 +
.../nativetask/testutil/BytesFactory.java | 121 +
.../EnforceNativeOutputCollectorDelegator.java | 51 +
.../nativetask/testutil/MockValueClass.java | 80 +
.../nativetask/testutil/ResultVerifier.java | 162 +
.../testutil/ScenarioConfiguration.java | 58 +
.../nativetask/testutil/TestConstants.java | 71 +
.../mapred/nativetask/testutil/TestInput.java | 61 +
.../mapred/nativetask/utils/TestBytesUtil.java | 69 +
.../nativetask/utils/TestReadWriteBuffer.java | 62 +
.../nativetask/utils/TestSizedWritable.java | 34 +
.../src/test/resources/common_conf.xml | 48 +
.../src/test/resources/kvtest-conf.xml | 85 +
.../src/test/resources/log4j.properties | 19 +
.../src/test/resources/native_conf.xml | 30 +
.../src/test/resources/normal_conf.xml | 26 +
.../src/test/resources/test-combiner-conf.xml | 30 +
.../src/test/resources/test-compress-conf.xml | 50 +
.../src/test/resources/test-nonsort-conf.xml | 30 +
.../hadoop-mapreduce-client/pom.xml | 1 +
hadoop-mapreduce-project/pom.xml | 30 +-
220 files changed, 52881 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[26/50] [abbrv] git commit: MAPREDUCE-5977. Fix or suppress
native-task gcc warnings. Contributed by Manu Zhang.
Posted by ar...@apache.org.
MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad4524c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad4524c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad4524c
Branch: refs/heads/HDFS-6581
Commit: fad4524c85848b8efe27dcf4a3e3a9ac1725bd2a
Parents: 6d39367
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Aug 27 12:23:03 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Aug 27 12:25:42 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 2 +
.../src/CMakeLists.txt | 9 +-
.../src/main/native/gtest/gtest.h | 19541 -----------------
.../src/main/native/gtest/include/gtest/gtest.h | 19541 +++++++++++++++++
.../src/main/native/src/NativeTask.h | 2 -
.../src/main/native/src/codec/BlockCodec.cc | 4 +-
.../src/main/native/src/handler/BatchHandler.h | 4 +-
.../main/native/src/handler/CombineHandler.cc | 10 +-
.../src/main/native/src/lib/Buffers.h | 4 +-
.../src/main/native/src/lib/IFile.cc | 11 +-
.../main/native/src/lib/MapOutputCollector.cc | 14 +-
.../main/native/src/lib/MapOutputCollector.h | 4 +-
.../src/main/native/src/lib/Merge.cc | 4 +-
.../src/main/native/src/lib/PartitionBucket.h | 12 +-
.../src/main/native/src/lib/commons.h | 3 +
.../src/main/native/src/util/StringUtil.cc | 6 +-
.../src/main/native/test/TestCompressions.cc | 7 +-
.../src/main/native/test/TestIFile.cc | 4 +
.../src/main/native/test/lib/TestByteBuffer.cc | 2 +-
.../native/test/lib/TestFixSizeContainer.cc | 2 +-
.../src/main/native/test/lib/TestIterator.cc | 2 -
.../native/test/lib/TestMemBlockIterator.cc | 6 +-
.../src/main/native/test/lib/TestMemoryBlock.cc | 1 +
.../src/main/native/test/util/TestHash.cc | 2 +-
24 files changed, 19600 insertions(+), 19597 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 95f7858..4dc08cb 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -15,3 +15,5 @@ MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd)
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd)
+MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 77b6109..0ab99db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -90,7 +90,7 @@ SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
# primitive configs
set(PRFLAGS "-DSIMPLE_MEMCPY")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -Wall")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -fno-strict-aliasing -Wall -Wno-sign-compare")
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
@@ -150,7 +150,6 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
include_directories(
${GENERATED_JAVAH}
- ${D}
${D}/src
${D}/src/util
${D}/src/lib
@@ -160,6 +159,8 @@ include_directories(
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
+# add gtest as system library to suppress gcc warnings
+include_directories(SYSTEM ${D}/gtest/include)
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
@@ -220,8 +221,9 @@ add_dual_library(nativetask
target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
+add_library(gtest ${D}/gtest/gtest-all.cc)
+set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
add_executable(nttest
- ${D}/gtest/gtest-all.cc
${D}/test/lib/TestByteArray.cc
${D}/test/lib/TestByteBuffer.cc
${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
@@ -260,6 +262,7 @@ ENDIF()
target_link_libraries(nttest
nativetask_static
+ gtest
${NT_DEPEND_LIBRARY}
)
[03/50] [abbrv] git commit: MAPREDUCE-5994. Simplify ByteUtils and
fix failing test. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613003 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d468a924
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d468a924
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d468a924
Branch: refs/heads/HDFS-6581
Commit: d468a9244740fcdeacf23b3aa08ab1c457e050ca
Parents: 341695e
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jul 24 06:14:27 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jul 24 06:14:27 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/NativeRuntime.java | 10 +-
.../mapred/nativetask/util/BytesUtil.java | 148 +++----------------
.../mapred/nativetask/util/ConfigUtil.java | 5 +-
.../mapred/nativetask/util/ReadWriteBuffer.java | 6 +-
.../mapred/nativetask/utils/TestBytesUtil.java | 19 +--
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 4 +-
.../nativetask/testutil/BytesFactory.java | 23 +--
8 files changed, 55 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 18bf029..166d68d 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -2,3 +2,4 @@ Changes for Hadoop Native Map Output Collector
------------------------------------------------
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
+MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index 53b1acd..65ce652 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -20,20 +20,16 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import com.google.common.base.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Task.TaskReporter;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
import org.apache.hadoop.mapred.nativetask.util.SnappyUtil;
import org.apache.hadoop.util.VersionInfo;
@@ -93,7 +89,7 @@ public class NativeRuntime {
*/
public synchronized static long createNativeObject(String clazz) {
assertNativeLibraryLoaded();
- final long ret = JNICreateNativeObject(BytesUtil.toBytes(clazz));
+ final long ret = JNICreateNativeObject(clazz.getBytes(Charsets.UTF_8));
if (ret == 0) {
LOG.warn("Can't create NativeObject for class " + clazz + ", probably not exist.");
}
@@ -108,7 +104,7 @@ public class NativeRuntime {
*/
public synchronized static long registerLibrary(String libraryName, String clazz) {
assertNativeLibraryLoaded();
- final long ret = JNIRegisterModule(BytesUtil.toBytes(libraryName), BytesUtil.toBytes(clazz));
+ final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8), clazz.getBytes(Charsets.UTF_8));
if (ret != 0) {
LOG.warn("Can't create NativeObject for class " + clazz + ", probably not exist.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index 9653baa..d90ae8d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -18,107 +18,38 @@
package org.apache.hadoop.mapred.nativetask.util;
-import java.io.UnsupportedEncodingException;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
public class BytesUtil {
- public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
- public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
-
- public static byte[] toBytes(String str) {
- if (str == null) {
- return null;
- }
- try {
- return str.getBytes("utf-8");
- } catch (final UnsupportedEncodingException e) {
- throw new RuntimeException(e.getMessage());
- }
- }
-
- public static String fromBytes(byte[] data) {
- if (data == null) {
- return null;
- }
- try {
- return new String(data, "utf-8");
- } catch (final UnsupportedEncodingException e) {
- throw new RuntimeException(e.getMessage());
- }
- }
-
- /**
- * Converts a byte array to an int value
- * @param bytes byte array
- * @return the int value
- */
- public static int toInt(byte[] bytes) {
- return toInt(bytes, 0, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- * @param bytes byte array
- * @param offset offset into array
- * @param length length of int (has to be {@link #SIZEOF_INT})
- * @return the int value
- * @throws RuntimeException if length is not {@link #SIZEOF_INT} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static int toInt(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_INT || offset + length > bytes.length) {
- throw new RuntimeException(
- "toInt exception. length not equals to SIZE of Int or buffer overflow");
- }
- int n = 0;
- for (int i = offset; i< offset + length; i++) {
- n <<= 4;
- n ^= bytes[i] & 0xff;
- }
- return n;
- }
-
/**
- * Converts a byte array to a long value.
- * @param bytes array
- * @return the long value
- */
- public static long toLong(byte[] bytes) {
- return toLong(bytes, 0, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value.
+ * Converts a big-endian byte array to a long value.
*
* @param bytes array of bytes
* @param offset offset into array
- * @return the long value
*/
public static long toLong(byte[] bytes, int offset) {
- return toLong(bytes, offset, SIZEOF_LONG);
+ return Longs.fromBytes(bytes[offset],
+ bytes[offset + 1],
+ bytes[offset + 2],
+ bytes[offset + 3],
+ bytes[offset + 4],
+ bytes[offset + 5],
+ bytes[offset + 6],
+ bytes[offset + 7]);
}
/**
- * Converts a byte array to a long value.
- *
- * @param bytes array of bytes
- * @param offset offset into array
- * @param length length of data (must be {@link #SIZEOF_LONG})
- * @return the long value
- * @throws RuntimeException if length is not {@link #SIZEOF_LONG} or
- * if there's not enough room in the array at the offset indicated.
+ * Convert a big-endian integer from a byte array to a primitive value.
+ * @param bytes the array to parse from
+ * @param offset the offset in the array
*/
- public static long toLong(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_LONG || offset + length > bytes.length) {
- throw new RuntimeException(
- "toLong exception. length not equals to SIZE of Long or buffer overflow");
- }
- long l = 0;
- for (int i = offset; i < offset + length; i++) {
- l <<= 8;
- l ^= bytes[i] & 0xff;
- }
- return l;
+ public static int toInt(byte[] bytes, int offset) {
+ return Ints.fromBytes(bytes[offset],
+ bytes[offset + 1],
+ bytes[offset + 2],
+ bytes[offset + 3]);
}
/**
@@ -137,7 +68,7 @@ public class BytesUtil {
* @return Float made from passed byte array.
*/
public static float toFloat(byte [] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+ return Float.intBitsToFloat(toInt(bytes, offset));
}
/**
@@ -154,7 +85,7 @@ public class BytesUtil {
* @return Return double made from passed bytes.
*/
public static double toDouble(final byte [] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+ return Double.longBitsToDouble(toLong(bytes, offset));
}
/**
@@ -211,45 +142,12 @@ public class BytesUtil {
}
/**
- * Convert an int value to a byte array. Big-endian. Same as what DataOutputStream.writeInt
- * does.
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(int val) {
- byte [] b = new byte[4];
- for(int i = 3; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Convert a long value to a byte array using big-endian.
- *
- * @param val value to convert
- * @return the byte array
- */
- public static byte[] toBytes(long val) {
- byte [] b = new byte[8];
- for (int i = 7; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
* @param f float value
* @return the float represented as byte []
*/
public static byte [] toBytes(final float f) {
// Encode it as int
- return toBytes(Float.floatToRawIntBits(f));
+ return Ints.toByteArray(Float.floatToRawIntBits(f));
}
/**
@@ -261,7 +159,7 @@ public class BytesUtil {
*/
public static byte [] toBytes(final double d) {
// Encode it as a long
- return toBytes(Double.doubleToRawLongBits(d));
+ return Longs.toByteArray(Double.doubleToRawLongBits(d));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
index 50db44c..d7cf00a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
@@ -21,14 +21,15 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Charsets;
import org.apache.hadoop.conf.Configuration;
public class ConfigUtil {
public static byte[][] toBytes(Configuration conf) {
List<byte[]> nativeConfigs = new ArrayList<byte[]>();
for (Map.Entry<String, String> e : conf) {
- nativeConfigs.add(BytesUtil.toBytes(e.getKey()));
- nativeConfigs.add(BytesUtil.toBytes(e.getValue()));
+ nativeConfigs.add(e.getKey().getBytes(Charsets.UTF_8));
+ nativeConfigs.add(e.getValue().getBytes(Charsets.UTF_8));
}
return nativeConfigs.toArray(new byte[nativeConfigs.size()][]);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
index 5dcae14..dd73c1e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapred.nativetask.util;
+import com.google.common.base.Charsets;
+
public class ReadWriteBuffer {
private byte[] _buff;
private int _writePoint;
@@ -127,13 +129,13 @@ public class ReadWriteBuffer {
}
public void writeString(String str) {
- final byte[] bytes = BytesUtil.toBytes(str);
+ final byte[] bytes = str.getBytes(Charsets.UTF_8);
writeBytes(bytes, 0, bytes.length);
}
public String readString() {
final byte[] bytes = readBytes();
- return BytesUtil.fromBytes(bytes);
+ return new String(bytes, Charsets.UTF_8);
}
private void checkWriteSpaceAndResizeIfNecessary(int toBeWritten) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
index 8d74d63..57b77cd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapred.nativetask.utils;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
import junit.framework.Assert;
import junit.framework.TestCase;
@@ -24,27 +26,18 @@ import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
@SuppressWarnings({ "deprecation" })
public class TestBytesUtil extends TestCase {
-
- public void testBytesStringConversion() {
-
- final String str = "I am good!";
- final byte[] bytes = BytesUtil.toBytes(str);
-
- Assert.assertEquals(str, BytesUtil.fromBytes(bytes));
- }
-
public void testBytesIntConversion() {
final int a = 1000;
- final byte[] intBytes = BytesUtil.toBytes(a);
+ final byte[] intBytes = Ints.toByteArray(a);
- Assert.assertEquals(a, BytesUtil.toInt(intBytes));
+ Assert.assertEquals(a, BytesUtil.toInt(intBytes, 0));
}
public void testBytesLongConversion() {
final long l = 1000000L;
- final byte[] longBytes = BytesUtil.toBytes(l);
+ final byte[] longBytes = Longs.toByteArray(l);
- Assert.assertEquals(l, BytesUtil.toLong(longBytes));
+ Assert.assertEquals(l, BytesUtil.toLong(longBytes, 0));
}
public void testBytesFloatConversion() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index c6ec9c2..6d683f8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.mapred.nativetask.kvtest;
import java.io.IOException;
import java.util.zip.CRC32;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -63,7 +63,7 @@ public class KVJob {
resultlong += crc32.getValue();
}
final VTYPE V = null;
- context.write(key, (VTYPE) BytesFactory.newObject(BytesUtil.toBytes(resultlong), V.getClass().getName()));
+ context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d468a924/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
index 5ad35d6..5185371 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativetask.testutil;
import java.util.Random;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -39,17 +41,17 @@ public class BytesFactory {
public static Object newObject(byte[] seed, String className) {
r.setSeed(seed.hashCode());
if (className.equals(IntWritable.class.getName())) {
- return new IntWritable(BytesUtil.toInt(seed));
+ return new IntWritable(Ints.fromByteArray(seed));
} else if (className.equals(FloatWritable.class.getName())) {
return new FloatWritable(r.nextFloat());
} else if (className.equals(DoubleWritable.class.getName())) {
return new DoubleWritable(r.nextDouble());
} else if (className.equals(LongWritable.class.getName())) {
- return new LongWritable(BytesUtil.toLong(seed));
+ return new LongWritable(Longs.fromByteArray(seed));
} else if (className.equals(VIntWritable.class.getName())) {
- return new VIntWritable(BytesUtil.toInt(seed));
+ return new VIntWritable(Ints.fromByteArray(seed));
} else if (className.equals(VLongWritable.class.getName())) {
- return new VLongWritable(BytesUtil.toLong(seed));
+ return new VLongWritable(Longs.fromByteArray(seed));
} else if (className.equals(BooleanWritable.class.getName())) {
return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
} else if (className.equals(Text.class.getName())) {
@@ -75,24 +77,25 @@ public class BytesFactory {
public static <VTYPE> byte[] toBytes(VTYPE obj) {
final String className = obj.getClass().getName();
if (className.equals(IntWritable.class.getName())) {
- return BytesUtil.toBytes(((IntWritable) obj).get());
+ return Ints.toByteArray(((IntWritable) obj).get());
} else if (className.equals(FloatWritable.class.getName())) {
return BytesUtil.toBytes(((FloatWritable) obj).get());
} else if (className.equals(DoubleWritable.class.getName())) {
return BytesUtil.toBytes(((DoubleWritable) obj).get());
} else if (className.equals(LongWritable.class.getName())) {
- return BytesUtil.toBytes(((LongWritable) obj).get());
+ return Longs.toByteArray(((LongWritable) obj).get());
} else if (className.equals(VIntWritable.class.getName())) {
- return BytesUtil.toBytes(((VIntWritable) obj).get());
+ return Ints.toByteArray(((VIntWritable) obj).get());
} else if (className.equals(VLongWritable.class.getName())) {
- return BytesUtil.toBytes(((VLongWritable) obj).get());
+ return Longs.toByteArray(((VLongWritable) obj).get());
} else if (className.equals(BooleanWritable.class.getName())) {
return BytesUtil.toBytes(((BooleanWritable) obj).get());
} else if (className.equals(Text.class.getName())) {
- return BytesUtil.toBytes(((Text) obj).toString());
+ return ((Text)obj).copyBytes();
} else if (className.equals(ByteWritable.class.getName())) {
- return BytesUtil.toBytes(((ByteWritable) obj).get());
+ return Ints.toByteArray((int) ((ByteWritable) obj).get());
} else if (className.equals(BytesWritable.class.getName())) {
+ // TODO: copyBytes instead?
return ((BytesWritable) obj).getBytes();
} else {
return new byte[0];
[08/50] [abbrv] git commit: MAPREDUCE-5996. native-task: Rename
system tests into standard directory layout. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613004 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2cba48f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2cba48f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2cba48f
Branch: refs/heads/HDFS-6581
Commit: b2cba48f36c96f190f6a25e65291873be7aee322
Parents: d468a92
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jul 24 06:17:33 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jul 24 06:17:33 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../nativetask/combinertest/CombinerTest.java | 122 +++++++++++
.../combinertest/LargeKVCombinerTest.java | 96 +++++++++
.../combinertest/OldAPICombinerTest.java | 107 ++++++++++
.../nativetask/combinertest/WordCount.java | 89 ++++++++
.../combinertest/WordCountWithOldAPI.java | 65 ++++++
.../nativetask/compresstest/CompressMapper.java | 69 ++++++
.../nativetask/compresstest/CompressTest.java | 134 ++++++++++++
.../nativetask/kvtest/HashSumReducer.java | 48 +++++
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 97 +++++++++
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 181 ++++++++++++++++
.../mapred/nativetask/kvtest/LargeKVTest.java | 129 +++++++++++
.../mapred/nativetask/kvtest/TestInputFile.java | 213 +++++++++++++++++++
.../nativetask/nonsorttest/NonSortTest.java | 99 +++++++++
.../nativetask/nonsorttest/NonSortTestMR.java | 71 +++++++
.../nativetask/testutil/BytesFactory.java | 104 +++++++++
.../EnforceNativeOutputCollectorDelegator.java | 48 +++++
.../nativetask/testutil/MockValueClass.java | 72 +++++++
.../nativetask/testutil/ResultVerifier.java | 141 ++++++++++++
.../testutil/ScenarioConfiguration.java | 58 +++++
.../nativetask/testutil/TestConstants.java | 67 ++++++
.../test/java/system/data/testGlibcBugSpill.out | 2 -
.../nativetask/combinertest/CombinerTest.java | 122 -----------
.../combinertest/LargeKVCombinerTest.java | 96 ---------
.../combinertest/OldAPICombinerTest.java | 107 ----------
.../nativetask/combinertest/WordCount.java | 89 --------
.../combinertest/WordCountWithOldAPI.java | 65 ------
.../nativetask/compresstest/CompressMapper.java | 69 ------
.../nativetask/compresstest/CompressTest.java | 134 ------------
.../nativetask/kvtest/HashSumReducer.java | 48 -----
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 97 ---------
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 181 ----------------
.../mapred/nativetask/kvtest/LargeKVTest.java | 129 -----------
.../mapred/nativetask/kvtest/TestInputFile.java | 213 -------------------
.../nativetask/nonsorttest/NonSortTest.java | 99 ---------
.../nativetask/nonsorttest/NonSortTestMR.java | 71 -------
.../nativetask/testutil/BytesFactory.java | 104 ---------
.../EnforceNativeOutputCollectorDelegator.java | 48 -----
.../nativetask/testutil/MockValueClass.java | 72 -------
.../nativetask/testutil/ResultVerifier.java | 141 ------------
.../testutil/ScenarioConfiguration.java | 58 -----
.../nativetask/testutil/TestConstants.java | 67 ------
42 files changed, 2011 insertions(+), 2012 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 166d68d..cea5a76 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -3,3 +3,4 @@ Changes for Hadoop Native Map Output Collector
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
+MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
new file mode 100644
index 0000000..8a4aa6f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.combinertest;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
+import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
+import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CombinerTest {
+ private FileSystem fs;
+ private String inputpath;
+ private String nativeoutputpath;
+ private String hadoopoutputpath;
+
+ @Test
+ public void testWordCountCombiner() {
+ try {
+
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
+
+ final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
+ commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
+
+ final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
+
+ nativejob.waitForCompletion(true);
+
+ Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ normaljob.waitForCompletion(true);
+ Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
+ assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
+ nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+
+ } catch (final Exception e) {
+ e.printStackTrace();
+ assertEquals("run exception", true, false);
+ }
+ }
+
+ @Before
+ public void startUp() throws Exception {
+ final ScenarioConfiguration conf = new ScenarioConfiguration();
+ conf.addcombinerConf();
+
+ this.fs = FileSystem.get(conf);
+
+ this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+ TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
+
+ if (!fs.exists(new Path(inputpath))) {
+ new TestInputFile(
+ conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000),
+ Text.class.getName(),
+ Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
+ }
+
+ this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
+ TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
+ this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
+ TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
+ }
+
+ protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
+ throws Exception {
+ final Configuration conf = new Configuration(inputConf);
+ conf.set("fileoutputpath", outputpath);
+ final FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(new Path(outputpath))) {
+ fs.delete(new Path(outputpath));
+ }
+ fs.close();
+ final Job job = new Job(conf, jobname);
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(TokenizerMapper.class);
+ job.setCombinerClass(IntSumReducer.class);
+ job.setReducerClass(IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ FileInputFormat.addInputPath(job, new Path(inputpath));
+ FileOutputFormat.setOutputPath(job, new Path(outputpath));
+ return job;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
new file mode 100644
index 0000000..50953e0
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.combinertest;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class LargeKVCombinerTest {
+
+ @Test
+ public void testLargeValueCombiner(){
+ final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final int deafult_KVSize_Maximum = 1 << 22; // 4M
+ final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
+ deafult_KVSize_Maximum);
+ final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+ TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
+ final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
+ TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
+ final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
+ TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
+ try {
+ final FileSystem fs = FileSystem.get(normalConf);
+ for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
+
+ int max = i;
+ int min = Math.max(i / 4, max - 10);
+
+ System.out.println("===KV Size Test: min size: " + min + ", max size: " + max);
+
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ fs.delete(new Path(inputPath), true);
+ new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
+ 1000000), IntWritable.class.getName(),
+ Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
+
+ final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
+ final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
+
+ nativejob.waitForCompletion(true);
+ Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ normaljob.waitForCompletion(true);
+ Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
+
+ final String reason = "LargeKVCombinerTest failed with, min size: " + min
+ + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
+
+ assertEquals(reason, true, compareRet);
+// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
+// nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+ }
+ fs.close();
+ } catch (final Exception e) {
+ e.printStackTrace();
+ assertEquals("run exception", true, false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
new file mode 100644
index 0000000..5691e02
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.combinertest;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Counter;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OldAPICombinerTest {
+ private FileSystem fs;
+ private String inputpath;
+
+ @Test
+ public void testWordCountCombinerWithOldAPI() throws Exception {
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
+ final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
+ RunningJob nativeRunning = JobClient.runJob(nativeJob);
+
+ Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
+ normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
+ final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
+
+ RunningJob normalRunning = JobClient.runJob(normalJob);
+ Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+
+ assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+ }
+
+ @Before
+ public void startUp() throws Exception {
+ final ScenarioConfiguration conf = new ScenarioConfiguration();
+ conf.addcombinerConf();
+ this.fs = FileSystem.get(conf);
+ this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+ TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
+
+ if (!fs.exists(new Path(inputpath))) {
+ new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
+ Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
+ }
+ }
+
+ private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
+ throws Exception {
+ final JobConf jobConf = new JobConf(configuration);
+ final FileSystem fs = FileSystem.get(configuration);
+ if (fs.exists(new Path(output))) {
+ fs.delete(new Path(output), true);
+ }
+ fs.close();
+ jobConf.setJobName(name);
+ jobConf.setOutputKeyClass(Text.class);
+ jobConf.setOutputValueClass(IntWritable.class);
+ jobConf.setMapperClass(WordCountWithOldAPI.TokenizerMapperWithOldAPI.class);
+ jobConf.setCombinerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
+ jobConf.setReducerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
+
+ jobConf.setInputFormat(SequenceFileInputFormat.class);
+ jobConf.setOutputFormat(TextOutputFormat.class);
+
+ FileInputFormat.setInputPaths(jobConf, new Path(input));
+ FileOutputFormat.setOutputPath(jobConf, new Path(output));
+ return jobConf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
new file mode 100644
index 0000000..490b82b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.combinertest;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class WordCount {
+
+ private static Log LOG = LogFactory.getLog(WordCount.class);
+
+ public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
+
+ private final static IntWritable one = new IntWritable(1);
+ private final Text word = new Text();
+
+ @Override
+ public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+ final StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
+ }
+ }
+
+ public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ private final IntWritable result = new IntWritable();
+
+ @Override
+ public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
+ InterruptedException {
+ int sum = 0;
+ for (final IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(key, result);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ final Configuration conf = new Configuration();
+ final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ if (otherArgs.length != 2) {
+ System.err.println("Usage: wordcount <in> <out>");
+ System.exit(2);
+ }
+ final Job job = new Job(conf, conf.get(MRJobConfig.JOB_NAME, "word count"));
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(TokenizerMapper.class);
+ job.setCombinerClass(IntSumReducer.class);
+ job.setReducerClass(IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
+ System.exit(job.waitForCompletion(true) ? 0 : 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
new file mode 100644
index 0000000..a11ea91
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.combinertest;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+public class WordCountWithOldAPI {
+
+ public static class TokenizerMapperWithOldAPI extends MapReduceBase implements
+ Mapper<Object, Text, Text, IntWritable> {
+ private final static IntWritable one = new IntWritable(1);
+ private final Text word = new Text();
+
+ @Override
+ public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+ final StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ output.collect(word, one);
+ }
+ }
+ }
+
+ public static class IntSumReducerWithOldAPI extends MapReduceBase implements
+ Reducer<Text, IntWritable, Text, IntWritable> {
+ private final IntWritable result = new IntWritable();
+
+ @Override
+ public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output,
+ Reporter reporter) throws IOException {
+ int sum = 0;
+ while (values.hasNext()) {
+ sum += values.next().get();
+ }
+ result.set(sum);
+ output.collect(key, result);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
new file mode 100644
index 0000000..83c4794
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.compresstest;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+public class CompressMapper {
+ public static final String inputFile = "./compress/input.txt";
+ public static final String outputFileDir = "./compress/output/";
+
+ public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
+
+ @Override
+ protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
+ context.write(key, value);
+ }
+ }
+
+ public static Job getCompressJob(String jobname, Configuration conf) {
+ Job job = null;
+ try {
+ job = new Job(conf, jobname + "-CompressMapperJob");
+ job.setJarByClass(CompressMapper.class);
+ job.setMapperClass(TextCompressMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Text.class);
+ final Path outputpath = new Path(outputFileDir + jobname);
+ // if output file exists ,delete it
+ final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
+ if (hdfs.exists(outputpath)) {
+ hdfs.delete(outputpath);
+ }
+ hdfs.close();
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ FileInputFormat.addInputPath(job, new Path(inputFile));
+ FileOutputFormat.setOutputPath(job, outputpath);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ }
+ return job;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
new file mode 100644
index 0000000..0406375
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.compresstest;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CompressTest {
+
+ @Test
+ public void testSnappyCompress() throws Exception {
+ final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
+ conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
+ final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
+ job.waitForCompletion(true);
+
+ final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
+ hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
+ final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
+ hadoopjob.waitForCompletion(true);
+
+ final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
+ CompressMapper.outputFileDir + "hadoopsnappy");
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Test
+ public void testGzipCompress() throws Exception {
+ final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
+ conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
+ final Job job = CompressMapper.getCompressJob("nativegzip", conf);
+ job.waitForCompletion(true);
+
+ final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
+ hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
+ final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
+ hadoopjob.waitForCompletion(true);
+
+ final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
+ CompressMapper.outputFileDir + "hadoopgzip");
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Test
+ public void testBzip2Compress() throws Exception {
+ final Configuration nativeconf = ScenarioConfiguration.getNativeConfiguration();
+ nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
+ final Job nativejob = CompressMapper.getCompressJob("nativebzip2", nativeconf);
+ nativejob.waitForCompletion(true);
+
+ final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
+ hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
+ final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2", hadoopconf);
+ hadoopjob.waitForCompletion(true);
+
+ final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
+ CompressMapper.outputFileDir + "hadoopbzip2");
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Test
+ public void testLz4Compress() throws Exception {
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
+ final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
+ nativeJob.waitForCompletion(true);
+
+ final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
+ hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
+ final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
+ hadoopJob.waitForCompletion(true);
+ final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
+ CompressMapper.outputFileDir + "hadooplz4");
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Test
+ public void testDefaultCompress() throws Exception {
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
+ final Job nativeJob = CompressMapper.getCompressJob("nativedefault", nativeConf);
+ nativeJob.waitForCompletion(true);
+
+ final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
+ hadoopConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
+ final Job hadoopJob = CompressMapper.getCompressJob("hadoopdefault", hadoopConf);
+ hadoopJob.waitForCompletion(true);
+ final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativedefault",
+ CompressMapper.outputFileDir + "hadoopdefault");
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Before
+ public void startUp() throws Exception {
+ final ScenarioConfiguration conf = new ScenarioConfiguration();
+ final FileSystem fs = FileSystem.get(conf);
+ final Path path = new Path(CompressMapper.inputFile);
+ fs.delete(path);
+ if (!fs.exists(path)) {
+ new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
+ TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
+ Text.class.getName(), Text.class.getName(), conf)
+ .createSequenceTestFile(CompressMapper.inputFile);
+
+ }
+ fs.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
new file mode 100644
index 0000000..ce23780
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.kvtest;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Reducer;
+
+public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, IntWritable> {
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(os);
+
+ @Override
+ public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
+ int hashSum = 0;
+ for (final VTYPE val : values) {
+ if (val instanceof Writable) {
+ os.reset();
+ ((Writable) val).write(dos);
+ final int hash = Arrays.hashCode(os.toByteArray());
+ hashSum += hash;
+ }
+ }
+
+ context.write(key, new IntWritable(hashSum));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
new file mode 100644
index 0000000..6d683f8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.kvtest;
+
+import java.io.IOException;
+import java.util.zip.CRC32;
+
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+public class KVJob {
+ public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
+ public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
+ Job job = null;
+
+ public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
+ @Override
+ public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
+ context.write(key, value);
+ }
+ }
+
+ public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
+ public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
+ context.write(key, value);
+ }
+ }
+
+ public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
+
+ @Override
+ public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
+ long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
+ final CRC32 crc32 = new CRC32();
+ for (final VTYPE val : values) {
+ crc32.reset();
+ crc32.update(BytesFactory.toBytes(val));
+ resultlong += crc32.getValue();
+ }
+ final VTYPE V = null;
+ context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
+ }
+ }
+
+ public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath,
+ String outputpath) throws Exception {
+ job = new Job(conf, jobname);
+ job.setJarByClass(KVJob.class);
+ job.setMapperClass(KVJob.ValueMapper.class);
+ job.setOutputKeyClass(keyclass);
+ job.setMapOutputValueClass(valueclass);
+
+ if (conf.get(TestConstants.NATIVETASK_KVTEST_CREATEFILE).equals("true")) {
+ final FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path(inputpath), true);
+ fs.close();
+ final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
+ TestConstants.FILESIZE_KEY, "1000")),
+ keyclass.getName(), valueclass.getName(), conf);
+ testfile.createSequenceTestFile(inputpath);
+
+ }
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ FileInputFormat.addInputPath(job, new Path(inputpath));
+ FileOutputFormat.setOutputPath(job, new Path(outputpath));
+ }
+
+ public void runJob() throws Exception {
+
+ job.waitForCompletion(true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
new file mode 100644
index 0000000..1e08854
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.kvtest;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class KVTest {
+ private static Class<?>[] keyclasses = null;
+ private static Class<?>[] valueclasses = null;
+ private static String[] keyclassNames = null;
+ private static String[] valueclassNames = null;
+
+ private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
+ private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();
+ static {
+ nativekvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
+ hadoopkvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
+ }
+
+ @Parameters(name = "key:{0}\nvalue:{1}")
+ public static Iterable<Class<?>[]> data() {
+ final String valueclassesStr = nativekvtestconf
+ .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
+ System.out.println(valueclassesStr);
+ valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
+ // " "
+ final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
+ for (int i = 0; i < valueclassNames.length; i++) {
+ try {
+ if (valueclassNames[i].equals("")) {
+ continue;
+ }
+ tmpvalueclasses.add(Class.forName(valueclassNames[i]));
+ } catch (final ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+ valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
+ final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
+ System.out.println(keyclassesStr);
+ keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
+ // " "
+ final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
+ for (int i = 0; i < keyclassNames.length; i++) {
+ try {
+ if (keyclassNames[i].equals("")) {
+ continue;
+ }
+ tmpkeyclasses.add(Class.forName(keyclassNames[i]));
+ } catch (final ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+ keyclasses = tmpkeyclasses.toArray(new Class[tmpkeyclasses.size()]);
+ final Class<?>[][] kvgroup = new Class<?>[keyclassNames.length * valueclassNames.length][2];
+ for (int i = 0; i < keyclassNames.length; i++) {
+ final int tmpindex = i * valueclassNames.length;
+ for (int j = 0; j < valueclassNames.length; j++) {
+ kvgroup[tmpindex + j][0] = keyclasses[i];
+ kvgroup[tmpindex + j][1] = valueclasses[j];
+ }
+ }
+ return Arrays.asList(kvgroup);
+ }
+
+ private final Class<?> keyclass;
+ private final Class<?> valueclass;
+
+ public KVTest(Class<?> keyclass, Class<?> valueclass) {
+ this.keyclass = keyclass;
+ this.valueclass = valueclass;
+
+ }
+
+ @Test
+ public void testKVCompability() {
+
+ try {
+ final String nativeoutput = this.runNativeTest(
+ "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
+ final String normaloutput = this.runNormalTest(
+ "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
+ final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
+ final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+ + keyclass.getName()
+ + "/" + valueclass.getName();
+ if(compareRet){
+ final FileSystem fs = FileSystem.get(hadoopkvtestconf);
+ fs.delete(new Path(nativeoutput), true);
+ fs.delete(new Path(normaloutput), true);
+ fs.delete(new Path(input), true);
+ fs.close();
+ }
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ } catch (final IOException e) {
+ assertEquals("test run exception:", null, e);
+ } catch (final Exception e) {
+ assertEquals("test run exception:", null, e);
+ }
+ }
+
+ @Before
+ public void startUp() {
+
+ }
+
+ private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
+ final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+ + keyclass.getName()
+ + "/" + valueclass.getName();
+ final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ final FileSystem fs = FileSystem.get(nativekvtestconf);
+ fs.delete(new Path(outputpath));
+ fs.close();
+ nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
+ try {
+ final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
+ keyJob.runJob();
+ } catch (final Exception e) {
+ return "native testcase run time error.";
+ }
+ return outputpath;
+ }
+
+ private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
+ final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+ + keyclass.getName()
+ + "/" + valueclass.getName();
+ final String outputpath = hadoopkvtestconf
+ .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
+ + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ final FileSystem fs = FileSystem.get(hadoopkvtestconf);
+ fs.delete(new Path(outputpath));
+ fs.close();
+ hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
+ try {
+ final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
+ keyJob.runJob();
+ } catch (final Exception e) {
+ return "normal testcase run time error.";
+ }
+ return outputpath;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
new file mode 100644
index 0000000..900b058
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.kvtest;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.Test;
+
+public class LargeKVTest {
+
+ @Test
+ public void testKeySize() {
+ runKVSizeTests(Text.class, IntWritable.class);
+ }
+
+ @Test
+ public void testValueSize() {
+ runKVSizeTests(IntWritable.class, Text.class);
+ }
+
+ private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
+ static {
+ nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
+ nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
+ normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
+ normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
+ }
+
+ public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
+ if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
+ return;
+ }
+ final int deafult_KVSize_Maximum = 1 << 22; // 4M
+ final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
+ deafult_KVSize_Maximum);
+ try {
+
+ for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
+ int min = i / 4;
+ int max = i;
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+
+ System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
+ + keyClass.getName() + ", valueClass: " + valueClass.getName());
+
+ final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
+ valueClass, nativeConf);
+ final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
+ valueClass, normalConf);
+ final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
+ final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
+ + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
+ + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
+ assertEquals(reason, true, compareRet);
+ }
+ } catch (final Exception e) {
+ // TODO: handle exception
+ // assertEquals("test run exception:", null, e);
+ e.printStackTrace();
+ }
+ }
+
+ private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
+ throws Exception {
+ final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
+ + "/" + valueclass.getName();
+ final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
+ + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ final FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path(outputpath), true);
+ fs.close();
+ try {
+ final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
+ keyJob.runJob();
+ } catch (final Exception e) {
+ return "normal testcase run time error.";
+ }
+ return outputpath;
+ }
+
+ private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
+ throws IOException {
+ final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
+ + "/" + valueclass.getName();
+ final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ final FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path(outputpath), true);
+ fs.close();
+ try {
+ final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
+ keyJob.runJob();
+ } catch (final Exception e) {
+ return "normal testcase run time error.";
+ }
+ return outputpath;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
new file mode 100644
index 0000000..a194697
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.kvtest;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.VIntWritable;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+
+
+public class TestInputFile {
+
+ public static class KVSizeScope {
+ private static final int DefaultMinNum = 1;
+ private static final int DefaultMaxNum = 64;
+
+ public int minBytesNum;
+ public int maxBytesNum;
+
+ public KVSizeScope() {
+ this.minBytesNum = DefaultMinNum;
+ this.maxBytesNum = DefaultMaxNum;
+ }
+
+ public KVSizeScope(int min, int max) {
+ this.minBytesNum = min;
+ this.maxBytesNum = max;
+ }
+ }
+
+ private static HashMap<String, KVSizeScope> map = new HashMap<String, KVSizeScope>();
+
+ private byte[] databuf = null;
+ private final String keyClsName, valueClsName;
+ private int filesize = 0;
+ private int keyMaxBytesNum, keyMinBytesNum;
+ private int valueMaxBytesNum, valueMinBytesNum;
+ private SequenceFile.Writer writer = null;
+ Random r = new Random();
+ public static final int DATABUFSIZE = 1 << 22; // 4M
+
+ private enum State {
+ KEY, VALUE
+ };
+
+ static {
+ map.put(BooleanWritable.class.getName(), new KVSizeScope(1, 1));
+ map.put(DoubleWritable.class.getName(), new KVSizeScope(8, 8));
+ map.put(FloatWritable.class.getName(), new KVSizeScope(4, 4));
+ map.put(VLongWritable.class.getName(), new KVSizeScope(8, 8));
+ map.put(ByteWritable.class.getName(), new KVSizeScope(1, 1));
+ map.put(LongWritable.class.getName(), new KVSizeScope(8, 8));
+ map.put(VIntWritable.class.getName(), new KVSizeScope(4, 4));
+ map.put(IntWritable.class.getName(), new KVSizeScope(4, 4));
+ }
+
+ public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception {
+ this.filesize = filesize;
+ this.databuf = new byte[DATABUFSIZE];
+ this.keyClsName = keytype;
+ this.valueClsName = valuetype;
+ final int defaultMinBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MIN, 1);
+ final int defaultMaxBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX, 64);
+
+ if (map.get(keytype) != null) {
+ keyMinBytesNum = map.get(keytype).minBytesNum;
+ keyMaxBytesNum = map.get(keytype).maxBytesNum;
+ } else {
+ keyMinBytesNum = defaultMinBytes;
+ keyMaxBytesNum = defaultMaxBytes;
+ }
+
+ if (map.get(valuetype) != null) {
+ valueMinBytesNum = map.get(valuetype).minBytesNum;
+ valueMaxBytesNum = map.get(valuetype).maxBytesNum;
+ } else {
+ valueMinBytesNum = defaultMinBytes;
+ valueMaxBytesNum = defaultMaxBytes;
+ }
+ }
+
+ public void createSequenceTestFile(String filepath) throws Exception {
+ int FULL_BYTE_SPACE = 256;
+ createSequenceTestFile(filepath, FULL_BYTE_SPACE);
+ }
+
+ public void createSequenceTestFile(String filepath, int base) throws Exception {
+ createSequenceTestFile(filepath, base, (byte)0);
+ }
+
+ public void createSequenceTestFile(String filepath, int base, byte start) throws Exception {
+ System.out.println("create file " + filepath);
+ System.out.println(keyClsName + " " + valueClsName);
+ Class<?> tmpkeycls, tmpvaluecls;
+ try {
+ tmpkeycls = Class.forName(keyClsName);
+ } catch (final ClassNotFoundException e) {
+ throw new Exception("key class not found: ", e);
+ }
+ try {
+ tmpvaluecls = Class.forName(valueClsName);
+ } catch (final ClassNotFoundException e) {
+ throw new Exception("key class not found: ", e);
+ }
+ try {
+ final Path outputfilepath = new Path(filepath);
+ final ScenarioConfiguration conf= new ScenarioConfiguration();
+ final FileSystem hdfs = outputfilepath.getFileSystem(conf);
+ writer = new SequenceFile.Writer(hdfs, conf, outputfilepath, tmpkeycls, tmpvaluecls);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ }
+
+ int tmpfilesize = this.filesize;
+ while (tmpfilesize > DATABUFSIZE) {
+ nextRandomBytes(databuf, base, start);
+ final int size = flushBuf(DATABUFSIZE);
+ tmpfilesize -= size;
+ }
+ nextRandomBytes(databuf, base, start);
+ flushBuf(tmpfilesize);
+
+ if (writer != null) {
+ IOUtils.closeStream(writer);
+ } else {
+ throw new Exception("no writer to create sequenceTestFile!");
+ }
+ }
+
+ private void nextRandomBytes(byte[] buf, int base) {
+ nextRandomBytes(buf, base, (byte)0);
+ }
+
+ private void nextRandomBytes(byte[] buf, int base, byte start) {
+ r.nextBytes(buf);
+ for (int i = 0; i < buf.length; i++) {
+ buf[i] = (byte) ((buf[i] & 0xFF) % base + start);
+ }
+ }
+
+ private int flushBuf(int buflen) throws Exception {
+ final Random r = new Random();
+ int keybytesnum = 0;
+ int valuebytesnum = 0;
+ int offset = 0;
+
+ while (offset < buflen) {
+ final int remains = buflen - offset;
+ keybytesnum = keyMaxBytesNum;
+ if (keyMaxBytesNum != keyMinBytesNum) {
+ keybytesnum = keyMinBytesNum + r.nextInt(keyMaxBytesNum - keyMinBytesNum);
+ }
+
+ valuebytesnum = valueMaxBytesNum;
+ if (valueMaxBytesNum != valueMinBytesNum) {
+ valuebytesnum = valueMinBytesNum + r.nextInt(valueMaxBytesNum - valueMinBytesNum);
+ }
+
+ if (keybytesnum + valuebytesnum > remains) {
+ break;
+ }
+
+ final byte[] key = new byte[keybytesnum];
+ final byte[] value = new byte[valuebytesnum];
+
+ System.arraycopy(databuf, offset, key, 0, keybytesnum);
+ offset += keybytesnum;
+
+ System.arraycopy(databuf, offset, value, 0, valuebytesnum);
+ offset += valuebytesnum;
+
+ try {
+ writer.append(BytesFactory.newObject(key, this.keyClsName), BytesFactory.newObject(value, this.valueClsName));
+ } catch (final IOException e) {
+ e.printStackTrace();
+ throw new Exception("sequence file create failed", e);
+ }
+ }
+ return offset;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
new file mode 100644
index 0000000..b7b03e7
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.nonsorttest;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
+import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
+import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NonSortTest {
+
+ @Test
+ public void nonSortTest() throws Exception {
+ Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
+ nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
+ String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
+ String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
+ final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
+ nativeNonSort.waitForCompletion(true);
+
+ Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
+ normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
+ inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
+ outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
+ final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
+ hadoopWithSort.waitForCompletion(true);
+
+ final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
+ normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ }
+
+ @Before
+ public void startUp() throws Exception {
+ final ScenarioConfiguration configuration = new ScenarioConfiguration();
+ configuration.addNonSortTestConf();
+ final FileSystem fs = FileSystem.get(configuration);
+ final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
+ if (!fs.exists(path)) {
+ new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
+ Text.class.getName(), configuration).createSequenceTestFile(path.toString());
+ }
+ fs.close();
+ }
+
+ private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
+ final FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(new Path(outputpath))) {
+ fs.delete(new Path(outputpath), true);
+ }
+ fs.close();
+ final Job job = new Job(conf, jobName);
+ job.setJarByClass(NonSortTestMR.class);
+ job.setMapperClass(NonSortTestMR.Map.class);
+ job.setReducerClass(NonSortTestMR.KeyHashSumReduce.class);
+ job.setOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(IntWritable.class);
+ job.setOutputValueClass(LongWritable.class);
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.addInputPath(job, new Path(inputpath));
+ FileOutputFormat.setOutputPath(job, new Path(outputpath));
+ return job;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
new file mode 100644
index 0000000..4ca2449
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.nonsorttest;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+
+public class NonSortTestMR {
+
+ public static class Map extends Mapper<Object, Text, Text, IntWritable> {
+ private final static IntWritable one = new IntWritable(1);
+ private final Text word = new Text();
+
+ @Override
+ public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+ final String line = value.toString();
+ final StringTokenizer tokenizer = new StringTokenizer(line);
+ while (tokenizer.hasMoreTokens()) {
+ word.set(tokenizer.nextToken());
+ context.write(word, one);
+ }
+ }
+ }
+
+ public static class KeyHashSumReduce extends Reducer<Text, IntWritable, Text, LongWritable> {
+ long sum = 0;
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(os);
+
+ @Override
+ public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
+ InterruptedException {
+ for (final IntWritable val : values) {
+ os.reset();
+ key.write(dos);
+ final int hash = Arrays.hashCode(os.toByteArray());
+ sum += hash;
+ }
+ }
+
+ @Override
+ public void cleanup(Context context) throws IOException, InterruptedException {
+ context.write(new Text("NonSortTest"), new LongWritable(sum));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
new file mode 100644
index 0000000..5185371
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.VIntWritable;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
+
+
+public class BytesFactory {
+ public static Random r = new Random();
+
+ public static Object newObject(byte[] seed, String className) {
+ r.setSeed(seed.hashCode());
+ if (className.equals(IntWritable.class.getName())) {
+ return new IntWritable(Ints.fromByteArray(seed));
+ } else if (className.equals(FloatWritable.class.getName())) {
+ return new FloatWritable(r.nextFloat());
+ } else if (className.equals(DoubleWritable.class.getName())) {
+ return new DoubleWritable(r.nextDouble());
+ } else if (className.equals(LongWritable.class.getName())) {
+ return new LongWritable(Longs.fromByteArray(seed));
+ } else if (className.equals(VIntWritable.class.getName())) {
+ return new VIntWritable(Ints.fromByteArray(seed));
+ } else if (className.equals(VLongWritable.class.getName())) {
+ return new VLongWritable(Longs.fromByteArray(seed));
+ } else if (className.equals(BooleanWritable.class.getName())) {
+ return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
+ } else if (className.equals(Text.class.getName())) {
+ return new Text(BytesUtil.toStringBinary(seed));
+ } else if (className.equals(ByteWritable.class.getName())) {
+ return new ByteWritable(seed.length > 0 ? seed[0] : 0);
+ } else if (className.equals(BytesWritable.class.getName())) {
+ return new BytesWritable(seed);
+ } else if (className.equals(UTF8.class.getName())) {
+ return new UTF8(BytesUtil.toStringBinary(seed));
+ } else if (className.equals(MockValueClass.class.getName())) {
+ return new MockValueClass(seed);
+ } else {
+ return null;
+ }
+ }
+
+
+ public static <VTYPE> byte[] fromBytes(byte[] bytes) throws Exception {
+ throw new Exception("Not supported");
+ }
+
+ public static <VTYPE> byte[] toBytes(VTYPE obj) {
+ final String className = obj.getClass().getName();
+ if (className.equals(IntWritable.class.getName())) {
+ return Ints.toByteArray(((IntWritable) obj).get());
+ } else if (className.equals(FloatWritable.class.getName())) {
+ return BytesUtil.toBytes(((FloatWritable) obj).get());
+ } else if (className.equals(DoubleWritable.class.getName())) {
+ return BytesUtil.toBytes(((DoubleWritable) obj).get());
+ } else if (className.equals(LongWritable.class.getName())) {
+ return Longs.toByteArray(((LongWritable) obj).get());
+ } else if (className.equals(VIntWritable.class.getName())) {
+ return Ints.toByteArray(((VIntWritable) obj).get());
+ } else if (className.equals(VLongWritable.class.getName())) {
+ return Longs.toByteArray(((VLongWritable) obj).get());
+ } else if (className.equals(BooleanWritable.class.getName())) {
+ return BytesUtil.toBytes(((BooleanWritable) obj).get());
+ } else if (className.equals(Text.class.getName())) {
+ return ((Text)obj).copyBytes();
+ } else if (className.equals(ByteWritable.class.getName())) {
+ return Ints.toByteArray((int) ((ByteWritable) obj).get());
+ } else if (className.equals(BytesWritable.class.getName())) {
+ // TODO: copyBytes instead?
+ return ((BytesWritable) obj).getBytes();
+ } else {
+ return new byte[0];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
new file mode 100644
index 0000000..a0f7d64
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
+
+public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
+ private boolean nativetaskloaded = false;
+
+ @Override
+ public void init(Context context)
+ throws IOException, ClassNotFoundException {
+ try {
+ super.init(context);
+ nativetaskloaded = true;
+ } catch (final Exception e) {
+ nativetaskloaded = false;
+ System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void collect(K key, V value, int partition) throws IOException, InterruptedException {
+ if (this.nativetaskloaded) {
+ super.collect(key, value, partition);
+ } else {
+ // nothing to do.
+ }
+ }
+}
[38/50] [abbrv] git commit: MAPREDUCE-6069. Lint/style fixes and
removal of unused code. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683987be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683987be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683987be
Branch: refs/heads/HDFS-6581
Commit: 683987be7c160e67ddb8534eeb3c464bbe2796dd
Parents: 0032216
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Sep 3 13:07:24 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 5 10:44:38 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/LICENSE | 22 --
.../src/CMakeLists.txt | 6 -
.../hadoop/mapred/nativetask/Constants.java | 13 -
.../hadoop/mapred/nativetask/DataReceiver.java | 3 -
.../mapred/nativetask/ICombineHandler.java | 2 -
.../mapred/nativetask/INativeHandler.java | 8 -
.../mapred/nativetask/NativeBatchProcessor.java | 11 -
.../NativeMapOutputCollectorDelegator.java | 10 +-
.../hadoop/mapred/nativetask/NativeRuntime.java | 32 +-
.../hadoop/mapred/nativetask/Platform.java | 11 +-
.../nativetask/handlers/BufferPullee.java | 3 +-
.../nativetask/handlers/BufferPuller.java | 2 -
.../nativetask/handlers/BufferPushee.java | 6 +-
.../nativetask/handlers/BufferPusher.java | 3 +-
.../nativetask/handlers/CombinerHandler.java | 16 +-
.../mapred/nativetask/handlers/IDataLoader.java | 1 -
.../handlers/NativeCollectorOnlyHandler.java | 6 +-
.../serde/BytesWritableSerializer.java | 3 +-
.../mapred/nativetask/serde/IKVSerializer.java | 31 +-
.../mapred/nativetask/serde/KVSerializer.java | 6 +-
.../nativetask/serde/NativeSerialization.java | 11 +-
.../mapred/nativetask/util/BytesUtil.java | 2 +-
.../nativetask/util/LocalJobOutputFiles.java | 58 +---
.../nativetask/util/NativeTaskOutput.java | 55 +---
.../nativetask/util/NativeTaskOutputFiles.java | 72 ++---
.../mapred/nativetask/util/ReadWriteBuffer.java | 16 +-
.../src/main/native/cityhash/city.cc | 307 -------------------
.../src/main/native/cityhash/city.h | 90 ------
.../src/main/native/src/NativeTask.h | 137 ---------
.../src/main/native/src/codec/BlockCodec.cc | 5 +-
.../src/main/native/src/codec/BlockCodec.h | 2 +-
.../src/main/native/src/codec/GzipCodec.cc | 2 +-
.../src/main/native/src/codec/GzipCodec.h | 2 +-
.../src/main/native/src/codec/Lz4Codec.cc | 2 +-
.../src/main/native/src/codec/Lz4Codec.h | 2 +-
.../src/main/native/src/codec/SnappyCodec.cc | 2 +-
.../src/main/native/src/codec/SnappyCodec.h | 2 +-
.../native/src/handler/AbstractMapHandler.cc | 9 +-
.../src/main/native/src/handler/BatchHandler.cc | 9 +-
.../main/native/src/handler/CombineHandler.cc | 4 +-
.../main/native/src/handler/CombineHandler.h | 2 +-
.../src/handler/MCollectorOutputHandler.cc | 6 +-
.../src/main/native/src/lib/BufferStream.cc | 116 +------
.../src/main/native/src/lib/BufferStream.h | 39 +--
.../src/main/native/src/lib/Buffers.cc | 77 +----
.../src/main/native/src/lib/Buffers.h | 63 +---
.../src/main/native/src/lib/Combiner.cc | 73 -----
.../src/main/native/src/lib/Combiner.h | 18 +-
.../src/main/native/src/lib/Compressions.cc | 6 +-
.../src/main/native/src/lib/Compressions.h | 2 +-
.../src/main/native/src/lib/FileSystem.cc | 23 +-
.../src/main/native/src/lib/FileSystem.h | 2 +-
.../src/main/native/src/lib/IFile.cc | 6 +-
.../src/main/native/src/lib/IFile.h | 10 +-
.../src/main/native/src/lib/Iterator.cc | 6 +-
.../src/main/native/src/lib/Log.cc | 2 +-
.../main/native/src/lib/MapOutputCollector.cc | 58 ++--
.../main/native/src/lib/MapOutputCollector.h | 16 +-
.../src/main/native/src/lib/MapOutputSpec.cc | 4 +-
.../src/main/native/src/lib/MapOutputSpec.h | 4 +-
.../src/main/native/src/lib/MemoryBlock.cc | 24 +-
.../src/main/native/src/lib/MemoryPool.h | 4 +-
.../src/main/native/src/lib/Merge.cc | 5 +-
.../src/main/native/src/lib/Merge.h | 8 +-
.../src/main/native/src/lib/MinHeap.h | 2 +-
.../src/main/native/src/lib/NativeLibrary.cc | 7 +-
.../main/native/src/lib/NativeObjectFactory.cc | 19 +-
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 27 +-
.../src/main/native/src/lib/NativeTask.cc | 44 +--
.../src/main/native/src/lib/PartitionBucket.cc | 25 +-
.../src/main/native/src/lib/PartitionBucket.h | 16 +-
.../native/src/lib/PartitionBucketIterator.cc | 22 +-
.../native/src/lib/PartitionBucketIterator.h | 16 +-
.../src/main/native/src/lib/Path.cc | 2 +-
.../src/main/native/src/lib/SpillInfo.cc | 12 +-
.../src/main/native/src/lib/Streams.cc | 6 +-
.../src/main/native/src/lib/TaskCounters.cc | 3 +-
.../src/main/native/src/lib/TrackingCollector.h | 50 ---
.../src/main/native/src/lib/commons.h | 9 +-
.../src/main/native/src/lib/jniutils.cc | 8 +-
.../src/main/native/src/util/Checksum.cc | 2 +-
.../src/main/native/src/util/Hash.cc | 24 --
.../src/main/native/src/util/Hash.h | 73 -----
.../src/main/native/src/util/Random.cc | 4 +-
.../src/main/native/src/util/StringUtil.cc | 9 +-
.../src/main/native/src/util/SyncUtils.cc | 114 +------
.../src/main/native/src/util/SyncUtils.h | 228 --------------
.../src/main/native/src/util/Timer.cc | 7 +-
.../src/main/native/src/util/WritableUtils.cc | 6 +-
.../src/main/native/src/util/WritableUtils.h | 2 +-
.../src/main/native/test/TestCommand.cc | 9 +-
.../src/main/native/test/TestCompressions.cc | 14 +-
.../src/main/native/test/TestConfig.cc | 6 +-
.../src/main/native/test/TestCounter.cc | 8 +-
.../src/main/native/test/TestFileSystem.cc | 2 +-
.../src/main/native/test/TestIFile.cc | 15 +-
.../src/main/native/test/TestMain.cc | 21 +-
.../src/main/native/test/TestPrimitives.cc | 2 +-
.../src/main/native/test/TestSort.cc | 33 +-
.../src/main/native/test/lib/TestByteArray.cc | 2 +-
.../src/main/native/test/lib/TestByteBuffer.cc | 2 +-
.../lib/TestComparatorForDualPivotQuickSort.cc | 2 +-
.../native/test/lib/TestComparatorForStdSort.cc | 2 +-
.../native/test/lib/TestFixSizeContainer.cc | 2 +-
.../src/main/native/test/lib/TestIterator.cc | 4 +-
.../src/main/native/test/lib/TestKVBuffer.cc | 2 +-
.../native/test/lib/TestMemBlockIterator.cc | 11 +-
.../src/main/native/test/lib/TestMemoryBlock.cc | 8 +-
.../src/main/native/test/lib/TestMemoryPool.cc | 14 +-
.../main/native/test/lib/TestPartitionBucket.cc | 36 +--
.../src/main/native/test/lib/TestReadBuffer.cc | 6 +-
.../main/native/test/lib/TestReadWriteBuffer.cc | 2 +-
.../native/test/lib/TestTrackingCollector.cc | 37 ---
.../src/main/native/test/test_commons.cc | 4 +-
.../src/main/native/test/test_commons.h | 4 +-
.../src/main/native/test/util/TestHash.cc | 52 ----
.../src/main/native/test/util/TestSyncUtils.cc | 116 -------
.../nativetask/combinertest/CombinerTest.java | 9 +-
.../combinertest/LargeKVCombinerTest.java | 11 +-
.../combinertest/OldAPICombinerTest.java | 21 +-
.../nativetask/combinertest/WordCount.java | 3 +-
.../combinertest/WordCountWithOldAPI.java | 9 +-
.../nativetask/compresstest/CompressMapper.java | 6 +-
.../nativetask/handlers/TestCombineHandler.java | 7 +-
.../TestNativeCollectorOnlyHandler.java | 15 +-
.../nativetask/kvtest/HashSumReducer.java | 3 +-
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 17 +-
.../mapred/nativetask/kvtest/TestInputFile.java | 3 +-
.../nativetask/nonsorttest/NonSortTest.java | 14 +-
.../nativetask/nonsorttest/NonSortTestMR.java | 3 +-
.../nativetask/serde/TestKVSerializer.java | 18 +-
.../serde/TestNativeSerialization.java | 3 +-
.../EnforceNativeOutputCollectorDelegator.java | 3 +-
.../nativetask/testutil/ResultVerifier.java | 15 +-
.../mapred/nativetask/testutil/TestInput.java | 9 +-
136 files changed, 546 insertions(+), 2335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 279b960..ea17907 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -21,3 +21,4 @@ MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is suces
MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
+MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
index 612fdf9..0780741 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
@@ -204,28 +204,6 @@
Licenses for third party projects used by this project:
-CityHash src/main/native/cityhash
----------------------------------------------------------------------
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
GoogleTest src/main/native/gtest
---------------------------------------------------------------------
Copyright 2008, Google Inc.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 0ab99db..7eed176 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -178,7 +178,6 @@ configure_file(main/native/test.sh test/test.sh)
add_dual_library(nativetask
${CMAKE_BINARY_DIR}/lz4.c
- ${D}/cityhash/city.cc
${D}/src/codec/BlockCodec.cc
${D}/src/codec/GzipCodec.cc
${D}/src/codec/Lz4Codec.cc
@@ -208,10 +207,8 @@ add_dual_library(nativetask
${D}/src/lib/SpillInfo.cc
${D}/src/lib/Path.cc
${D}/src/lib/Streams.cc
- ${D}/src/lib/Combiner.cc
${D}/src/lib/TaskCounters.cc
${D}/src/util/Checksum.cc
- ${D}/src/util/Hash.cc
${D}/src/util/Random.cc
${D}/src/util/StringUtil.cc
${D}/src/util/SyncUtils.cc
@@ -237,11 +234,8 @@ add_executable(nttest
${D}/test/lib/TestPartitionBucket.cc
${D}/test/lib/TestReadBuffer.cc
${D}/test/lib/TestReadWriteBuffer.cc
- ${D}/test/lib/TestTrackingCollector.cc
${D}/test/util/TestChecksum.cc
- ${D}/test/util/TestHash.cc
${D}/test/util/TestStringUtil.cc
- ${D}/test/util/TestSyncUtils.cc
${D}/test/util/TestWritableUtils.cc
${D}/test/TestCommand.cc
${D}/test/TestConfig.cc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
index ff1a95c..1f7a5d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
@@ -25,22 +25,9 @@ public class Constants {
public static final String MAP_SORT_CLASS = "map.sort.class";
public static final String MAPRED_COMBINER_CLASS = "mapred.combiner.class";
- public static final String MAPRED_MAPTASK_DELEGATOR_CLASS = "mapreduce.map.task.delegator.class";
- public static final String MAPRED_REDUCETASK_DELEGATOR_CLASS = "mapreduce.reduce.task.delegator.class";
- public static final String NATIVE_TASK_ENABLED = "native.task.enabled";
public static final String NATIVE_LOG_DEVICE = "native.log.device";
public static final String NATIVE_HADOOP_VERSION = "native.hadoop.version";
- public static final String NATIVE_MAPPER_CLASS = "native.mapper.class";
- public static final String NATIVE_REDUCER_CLASS = "native.reducer.class";
- public static final String NATIVE_PARTITIONER_CLASS = "native.partitioner.class";
- public static final String NATIVE_COMBINER_CLASS = "native.combiner.class";
- public static final String NATIVE_INPUT_SPLIT = "native.input.split";
-
- public static final String NATIVE_RECORDREADER_CLASS = "native.recordreader.class";
- public static final String NATIVE_RECORDWRITER_CLASS = "native.recordwriter.class";
- public static final String NATIVE_OUTPUT_FILE_NAME = "native.output.file.name";
-
public static final String NATIVE_PROCESSOR_BUFFER_KB = "native.processor.buffer.kb";
public static final int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64;
public static final int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
index 85f78ca..b3bdf53 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
@@ -32,9 +32,6 @@ public interface DataReceiver {
/**
* Send a signal to the receiver that the data arrives.
* The data is transferred in another band.
- *
- * @return
- * @throws IOException
*/
public boolean receiveData() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
index e4e706b..d28b883c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
@@ -29,7 +29,6 @@ public interface ICombineHandler {
/**
* run combiner
- * @throws IOException
*/
public void combine() throws IOException;
@@ -40,7 +39,6 @@ public interface ICombineHandler {
/**
* close handlers, buffer pullers and pushers
- * @throws IOException
*/
public void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
index 9506c7f..fef56d9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
@@ -45,17 +45,9 @@ public interface INativeHandler extends NativeDataTarget, NativeDataSource {
/**
* call command to downstream
- *
- * @param command
- * @param parameter
- * @return
- * @throws IOException
*/
public ReadWriteBuffer call(Command command, ReadWriteBuffer parameter) throws IOException;
- /**
- * @param handler
- */
void setCommandDispatcher(CommandDispatcher handler);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index 3c3811f..9458c6b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -229,32 +229,21 @@ public class NativeBatchProcessor implements INativeHandler {
/**
* Let native side to process data in inputBuffer
- *
- * @param handler
- * @param length
*/
private native void nativeProcessInput(long handler, int length);
/**
* Notice native side input is finished
- *
- * @param handler
*/
private native void nativeFinish(long handler);
/**
* Send control message to native side
- *
- * @param cmd
- * command data
- * @return return value
*/
private native byte[] nativeCommand(long handler, int cmd, byte[] parameter);
/**
* Load data from native
- *
- * @return
*/
private native void nativeLoadData(long handler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 828d7df..5a516c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -95,7 +95,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) {
- String message = "Native-Task doesn't support sort class " + job.get(Constants.MAP_SORT_CLASS);
+ String message = "Native-Task doesn't support sort class " +
+ job.get(Constants.MAP_SORT_CLASS);
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -115,8 +116,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
LOG.error(message);
throw new InvalidJobConfException(message);
} else if (!Platforms.support(keyCls.getName(), serializer, job)) {
- String message = "Native output collector doesn't support this key, this key is not comparable in native "
- + keyCls.getName();
+ String message = "Native output collector doesn't support this key, " +
+ "this key is not comparable in native: " + keyCls.getName();
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -144,7 +145,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
updater.start();
} else {
- String message = "Nativeruntime cannot be loaded, please check the libnativetask.so is in hadoop library dir";
+ String message = "NativeRuntime cannot be loaded, please check that " +
+ "libnativetask.so is in hadoop library dir";
LOG.error(message);
throw new InvalidJobConfException(message);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index 395cc89..a0e88bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -79,9 +79,6 @@ public class NativeRuntime {
/**
* create native object We use it to create native handlers
- *
- * @param clazz
- * @return
*/
public synchronized static long createNativeObject(String clazz) {
assertNativeLibraryLoaded();
@@ -94,13 +91,11 @@ public class NativeRuntime {
/**
* Register a customized library
- *
- * @param clazz
- * @return
*/
public synchronized static long registerLibrary(String libraryName, String clazz) {
assertNativeLibraryLoaded();
- final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8), clazz.getBytes(Charsets.UTF_8));
+ final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8),
+ clazz.getBytes(Charsets.UTF_8));
if (ret != 0) {
LOG.warn("Can't create NativeObject for class " + clazz + ", probably not exist.");
}
@@ -117,9 +112,6 @@ public class NativeRuntime {
/**
* Get the status report from native space
- *
- * @param reporter
- * @throws IOException
*/
public static void reportStatus(TaskReporter reporter) throws IOException {
assertNativeLibraryLoaded();
@@ -164,40 +156,32 @@ public class NativeRuntime {
/**
* Config the native runtime with mapreduce job configurations.
- *
- * @param configs
*/
private native static void JNIConfigure(byte[][] configs);
/**
* create a native object in native space
- *
- * @param clazz
- * @return
*/
private native static long JNICreateNativeObject(byte[] clazz);
/**
* create the default native object for certain type
- *
- * @param type
- * @return
*/
@Deprecated
private native static long JNICreateDefaultNativeObject(byte[] type);
/**
* destroy native object in native space
- *
- * @param addr
*/
private native static void JNIReleaseNativeObject(long addr);
/**
- * get status update from native side Encoding: progress:float status:Text Counter number: int the count of the
- * counters Counters: array [group:Text, name:Text, incrCount:Long]
- *
- * @return
+ * Get status update from native side
+ * Encoding:
+ * progress:float
+ * status:Text
+ * number: int the count of the counters
+ * Counters: array [group:Text, name:Text, incrCount:Long]
*/
private native static byte[] JNIUpdateStatus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
index 7e41989..efec26c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
@@ -49,8 +49,6 @@ public abstract class Platform {
/**
* initialize a platform, where we should call registerKey
- *
- * @throws IOException
*/
public abstract void init() throws IOException;
@@ -65,7 +63,6 @@ public abstract class Platform {
*
* @param keyClassName map out key class name
* @param key key serializer class
- * @throws IOException
*/
protected void registerKey(String keyClassName, Class<?> key) throws IOException {
serialization.register(keyClassName, key);
@@ -92,12 +89,12 @@ public abstract class Platform {
/**
* whether it's the platform that has defined a custom Java comparator
*
- * NativeTask doesn't support custom Java comparator(set with mapreduce.job.output.key.comparator.class)
- * but a platform (e.g Pig) could also set that conf and implement native comparators so
- * we shouldn't bail out.
+ * NativeTask doesn't support custom Java comparators
+ * (set with mapreduce.job.output.key.comparator.class)
+ * but a platform (e.g Pig) could also set that conf and implement native
+ * comparators so we shouldn't bail out.
*
* @param keyComparator comparator set with mapreduce.job.output.key.comparator.class
- * @return
*/
protected abstract boolean define(Class<?> keyComparator);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
index 1047bbf..9fd7a04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
@@ -46,7 +46,8 @@ public class BufferPullee<IK, IV> implements IDataLoader {
private final NativeDataTarget target;
private boolean closed = false;
- public BufferPullee(Class<IK> iKClass, Class<IV> iVClass, RawKeyValueIterator rIter, NativeDataTarget target)
+ public BufferPullee(Class<IK> iKClass, Class<IV> iVClass,
+ RawKeyValueIterator rIter, NativeDataTarget target)
throws IOException {
this.rIter = rIter;
tmpInputKey = new SizedWritable<IK>(iKClass);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
index 18523ef..b5dff92 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
@@ -189,8 +189,6 @@ public class BufferPuller implements RawKeyValueIterator, DataReceiver {
/**
* Closes the iterator so that the underlying streams can be closed.
- *
- * @throws IOException
*/
@Override
public void close() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
index fe78325..090882d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
@@ -51,7 +51,8 @@ public class BufferPushee<OK, OV> implements Closeable {
private KVSerializer<OK, OV> deserializer;
private boolean closed = false;
- public BufferPushee(Class<OK> oKClass, Class<OV> oVClass, RecordWriter<OK, OV> writer) throws IOException {
+ public BufferPushee(Class<OK> oKClass, Class<OV> oVClass,
+ RecordWriter<OK, OV> writer) throws IOException {
tmpOutputKey = new SizedWritable<OK>(oKClass);
tmpOutputValue = new SizedWritable<OV>(oVClass);
@@ -128,7 +129,8 @@ public class BufferPushee<OK, OV> implements Closeable {
}
}
if (remain != totalRead) {
- throw new IOException("We expect to read " + remain + ", but we actually read: " + totalRead);
+ throw new IOException("We expect to read " + remain +
+ ", but we actually read: " + totalRead);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
index 10d619b..454127c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
@@ -44,7 +44,8 @@ public class BufferPusher<K, V> implements OutputCollector<K, V> {
IKVSerializer serializer;
private boolean closed = false;
- public BufferPusher(Class<K> iKClass, Class<V> iVClass, NativeDataTarget target) throws IOException {
+ public BufferPusher(Class<K> iKClass, Class<V> iVClass,
+ NativeDataTarget target) throws IOException {
tmpInputKey = new SizedWritable<K>(iKClass);
tmpInputValue = new SizedWritable<V>(iVClass);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
index dad7af9..fcaeca1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
@@ -49,7 +49,8 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
private final BufferPusher<K, V> kvPusher;
private boolean closed = false;
- public static <K, V> ICombineHandler create(TaskContext context) throws IOException, ClassNotFoundException {
+ public static <K, V> ICombineHandler create(TaskContext context)
+ throws IOException, ClassNotFoundException {
final JobConf conf = new JobConf(context.getConf());
conf.set(Constants.SERIALIZATION_FRAMEWORK,
String.valueOf(SerializationFramework.WRITABLE_SERIALIZATION.getType()));
@@ -66,11 +67,13 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
final Counter combineInputCounter = context.getTaskReporter().getCounter(
TaskCounter.COMBINE_INPUT_RECORDS);
-
- final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(conf, context.getTaskAttemptId(),
+
+ final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(
+ conf, context.getTaskAttemptId(),
combineInputCounter, context.getTaskReporter(), null);
- final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, conf, DataChannel.INOUT);
+ final INativeHandler nativeHandler = NativeBatchProcessor.create(
+ NAME, conf, DataChannel.INOUT);
@SuppressWarnings("unchecked")
final BufferPusher<K, V> pusher = new BufferPusher<K, V>((Class<K>)context.getInputKeyClass(),
(Class<V>)context.getInputValueClass(),
@@ -79,8 +82,9 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
return new CombinerHandler<K, V>(nativeHandler, combinerRunner, puller, pusher);
}
- public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner, BufferPuller puller,
- BufferPusher<K, V> kvPusher) throws IOException {
+ public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner,
+ BufferPuller puller, BufferPusher<K, V> kvPusher)
+ throws IOException {
this.nativeHandler = nativeHandler;
this.combinerRunner = combiner;
this.puller = puller;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
index d2cbf1f..6eaf28b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
@@ -30,7 +30,6 @@ public interface IDataLoader {
/**
* @return size of data loaded
- * @throws IOException
*/
public int load() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
index 284c3dc..34238ae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
@@ -63,7 +63,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
private final INativeHandler nativeHandler;
private boolean closed = false;
- public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context) throws IOException {
+ public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context)
+ throws IOException {
ICombineHandler combinerHandler = null;
@@ -81,7 +82,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
LOG.info("[NativeCollectorOnlyHandler] combiner is not null");
}
- final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, context.getConf(), DataChannel.OUT);
+ final INativeHandler nativeHandler = NativeBatchProcessor.create(
+ NAME, context.getConf(), DataChannel.OUT);
final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>(
(Class<K>)context.getOutputKeyClass(),
(Class<V>)context.getOutputValueClass(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
index 08264f9..48dccfa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
@InterfaceAudience.Private
-public class BytesWritableSerializer implements INativeComparable, INativeSerializer<BytesWritable> {
+public class BytesWritableSerializer
+ implements INativeComparable, INativeSerializer<BytesWritable> {
@Override
public int getLength(BytesWritable w) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
index 8506d17..15a2ef7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
@@ -33,43 +33,16 @@ public interface IKVSerializer {
/**
* update the length field of SizedWritable
- * @param key
- * @param value
- * @throws IOException
*/
public void updateLength(SizedWritable<?> key, SizedWritable<?> value) throws IOException;
- /**
- *
- * @param out
- * @param key
- * @param value
- * @return bytes written
- * @throws IOException
- */
public int serializeKV(DataOutputStream out, SizedWritable<?> key,
SizedWritable<?> value) throws IOException;
- /**
- * serialize partitionId as well
- * @param out
- * @param partitionId
- * @param key
- * @param value
- * @return
- * @throws IOException
- */
public int serializePartitionKV(DataOutputStream out, int partitionId,
SizedWritable<?> key, SizedWritable<?> value)
throws IOException;
- /**
- *
- * @param in
- * @param key
- * @param value
- * @return bytes read
- * @throws IOException
- */
- public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value) throws IOException;
+ public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
index d650a4d..e6609b8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
@@ -55,7 +55,8 @@ public class KVSerializer<K, V> implements IKVSerializer {
}
@Override
- public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) throws IOException {
+ public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value)
+ throws IOException {
return serializePartitionKV(out, -1, key, value);
}
@@ -64,7 +65,8 @@ public class KVSerializer<K, V> implements IKVSerializer {
SizedWritable<?> key, SizedWritable<?> value)
throws IOException {
- if (key.length == SizedWritable.INVALID_LENGTH || value.length == SizedWritable.INVALID_LENGTH) {
+ if (key.length == SizedWritable.INVALID_LENGTH ||
+ value.length == SizedWritable.INVALID_LENGTH) {
updateLength(key, value);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
index 5881a46..11bde64 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.io.Writable;
@InterfaceAudience.Private
public class NativeSerialization {
- private final ConcurrentHashMap<String, Class<?>> map = new ConcurrentHashMap<String, Class<?>>();
+ private final ConcurrentHashMap<String, Class<?>> map =
+ new ConcurrentHashMap<String, Class<?>>();
public boolean accept(Class<?> c) {
return Writable.class.isAssignableFrom(c);
@@ -40,7 +41,8 @@ public class NativeSerialization {
return null;
}
if (!Writable.class.isAssignableFrom(c)) {
- throw new IOException("Cannot serialize type " + c.getName() + ", we only accept subclass of Writable");
+ throw new IOException("Cannot serialize type " + c.getName() +
+ ", we only accept subclass of Writable");
}
final String name = c.getName();
final Class<?> serializer = map.get(name);
@@ -70,8 +72,9 @@ public class NativeSerialization {
return;
} else {
if (!storedSerializer.getName().equals(serializer.getName())) {
- throw new IOException("Error! Serializer already registered, exist: " + storedSerializer.getName() + ", new: "
- + serializer.getName());
+ throw new IOException("Error! Serializer already registered, existing: " +
+ storedSerializer.getName() + ", new: " +
+ serializer.getName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index 35ef0a9..16f6919 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -97,7 +97,7 @@ public class BytesUtil {
* Write a printable representation of a byte array.
*
* @param b byte array
- * @return string
+ * @return the printable presentation
* @see #toStringBinary(byte[], int, int)
*/
public static String toStringBinary(final byte [] b) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
index 6d63be5..aa97789 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
@@ -45,9 +45,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -57,10 +54,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map output file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -69,9 +63,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -81,10 +72,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -94,10 +82,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber);
@@ -107,12 +92,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber);
@@ -122,10 +103,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException {
String path = String
@@ -136,12 +114,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException {
String path = String
@@ -152,10 +126,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException {
return lDirAlloc.getLocalPathToRead(
@@ -166,14 +137,11 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
- public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException {
+ public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf)
+ throws IOException {
return lDirAlloc.getLocalPathForWrite(
String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size,
conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
index e92901b..7c9f433 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
@@ -32,103 +32,70 @@ public interface NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException;
/**
* Create a local map output file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException;
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException;
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException;
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException;
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException;
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException;
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ r* @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException;
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException;
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
index 5cad10b..d52ee1c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
@@ -28,11 +28,13 @@ import org.apache.hadoop.mapred.TaskID;
/**
* Manipulate the working area for the transient store for maps and reduces.
- *
- * This class is used by map and reduce tasks to identify the directories that they need to write
- * to/read from for intermediate files. The callers of these methods are from child space and see
- * mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used
- * from TaskTracker space.
+ *
+ * This class is used by map and reduce tasks to identify the directories that they need
+ * to write to/read from for intermediate files. The callers of these methods are from
+ * child space and see mapreduce.cluster.local.dir as
+ * taskTracker/jobCache/jobId/attemptId.
+ *
+ * This class should not be used from TaskTracker space.
*/
@InterfaceAudience.Private
public class NativeTaskOutputFiles implements NativeTaskOutput {
@@ -55,9 +57,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -66,11 +65,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map output file name.
- *
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ *
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -79,9 +75,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -91,10 +84,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -104,10 +94,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber);
@@ -117,12 +104,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber);
@@ -132,10 +115,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException {
String path = String
@@ -146,12 +126,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException {
String path = String
@@ -162,10 +138,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException {
return lDirAlloc.getLocalPathToRead(
@@ -176,14 +149,11 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
- public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException {
+ public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf)
+ throws IOException {
return lDirAlloc.getLocalPathForWrite(
String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size,
conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
index 017d834..03df016 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
@@ -112,10 +112,15 @@ public class ReadWriteBuffer {
}
public long readLong() {
- final long result = ((_buff[_readPoint + 0] & 255) << 0) + ((_buff[_readPoint + 1] & 255) << 8)
- + ((_buff[_readPoint + 2] & 255) << 16) + ((long) (_buff[_readPoint + 3] & 255) << 24)
- + ((long) (_buff[_readPoint + 4] & 255) << 32) + ((long) (_buff[_readPoint + 5] & 255) << 40)
- + ((long) (_buff[_readPoint + 6] & 255) << 48) + (((long) _buff[_readPoint + 7] << 56));
+ final long result =
+ ((_buff[_readPoint + 0] & 255) << 0) +
+ ((_buff[_readPoint + 1] & 255) << 8) +
+ ((_buff[_readPoint + 2] & 255) << 16) +
+ ((long) (_buff[_readPoint + 3] & 255) << 24) +
+ ((long) (_buff[_readPoint + 4] & 255) << 32) +
+ ((long) (_buff[_readPoint + 5] & 255) << 40) +
+ ((long) (_buff[_readPoint + 6] & 255) << 48) +
+ (((long) _buff[_readPoint + 7] << 56));
_readPoint += 8;
return result;
@@ -144,7 +149,8 @@ public class ReadWriteBuffer {
if (_buff.length - _writePoint >= toBeWritten) {
return;
}
- final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ? (toBeWritten + _writePoint) : CACHE_LINE_SIZE;
+ final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ?
+ (toBeWritten + _writePoint) : CACHE_LINE_SIZE;
final byte[] newBuff = new byte[newLength];
System.arraycopy(_buff, 0, newBuff, 0, _writePoint);
_buff = newBuff;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
deleted file mode 100644
index 36ff93b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
+++ /dev/null
@@ -1,307 +0,0 @@
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-//
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
-//
-// This file provides CityHash64() and related functions.
-//
-// It's probably possible to create even faster hash functions by
-// writing a program that systematically explores some of the space of
-// possible hash functions, by using SIMD instructions, or by
-// compromising on hash quality.
-
-#include "city.h"
-
-#include <algorithm>
-
-using namespace std;
-
-#define UNALIGNED_LOAD64(p) (*(const uint64*)(p))
-#define UNALIGNED_LOAD32(p) (*(const uint32*)(p))
-
-#if !defined(LIKELY)
-#if defined(__GNUC__)
-#define LIKELY(x) (__builtin_expect(!!(x), 1))
-#else
-#define LIKELY(x) (x)
-#endif
-#endif
-
-// Some primes between 2^63 and 2^64 for various uses.
-static const uint64 k0 = 0xc3a5c85c97cb3127ULL;
-static const uint64 k1 = 0xb492b66fbe98f273ULL;
-static const uint64 k2 = 0x9ae16a3b2f90404fULL;
-static const uint64 k3 = 0xc949d7c7509e6557ULL;
-
-// Bitwise right rotate. Normally this will compile to a single
-// instruction, especially if the shift is a manifest constant.
-static uint64 Rotate(uint64 val, int shift) {
- // Avoid shifting by 64: doing so yields an undefined result.
- return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
-}
-
-// Equivalent to Rotate(), but requires the second arg to be non-zero.
-// On x86-64, and probably others, it's possible for this to compile
-// to a single instruction if both args are already in registers.
-static uint64 RotateByAtLeast1(uint64 val, int shift) {
- return (val >> shift) | (val << (64 - shift));
-}
-
-static uint64 ShiftMix(uint64 val) {
- return val ^ (val >> 47);
-}
-
-static uint64 HashLen16(uint64 u, uint64 v) {
- return Hash128to64(uint128(u, v));
-}
-
-static uint64 HashLen0to16(const char *s, size_t len) {
- if (len > 8) {
- uint64 a = UNALIGNED_LOAD64(s);
- uint64 b = UNALIGNED_LOAD64(s + len - 8);
- return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b;
- }
- if (len >= 4) {
- uint64 a = UNALIGNED_LOAD32(s);
- return HashLen16(len + (a << 3), UNALIGNED_LOAD32(s + len - 4));
- }
- if (len > 0) {
- uint8 a = s[0];
- uint8 b = s[len >> 1];
- uint8 c = s[len - 1];
- uint32 y = static_cast<uint32>(a) + (static_cast<uint32>(b) << 8);
- uint32 z = len + (static_cast<uint32>(c) << 2);
- return ShiftMix(y * k2 ^ z * k3) * k2;
- }
- return k2;
-}
-
-// This probably works well for 16-byte strings as well, but it may be overkill
-// in that case.
-static uint64 HashLen17to32(const char *s, size_t len) {
- uint64 a = UNALIGNED_LOAD64(s) * k1;
- uint64 b = UNALIGNED_LOAD64(s + 8);
- uint64 c = UNALIGNED_LOAD64(s + len - 8) * k2;
- uint64 d = UNALIGNED_LOAD64(s + len - 16) * k0;
- return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d,
- a + Rotate(b ^ k3, 20) - c + len);
-}
-
-// Return a 16-byte hash for 48 bytes. Quick and dirty.
-// Callers do best to use "random-looking" values for a and b.
-static pair<uint64, uint64> WeakHashLen32WithSeeds(
- uint64 w, uint64 x, uint64 y, uint64 z, uint64 a, uint64 b) {
- a += w;
- b = Rotate(b + a + z, 21);
- uint64 c = a;
- a += x;
- a += y;
- b += Rotate(a, 44);
- return make_pair(a + z, b + c);
-}
-
-// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty.
-static pair<uint64, uint64> WeakHashLen32WithSeeds(
- const char* s, uint64 a, uint64 b) {
- return WeakHashLen32WithSeeds(UNALIGNED_LOAD64(s),
- UNALIGNED_LOAD64(s + 8),
- UNALIGNED_LOAD64(s + 16),
- UNALIGNED_LOAD64(s + 24),
- a,
- b);
-}
-
-// Return an 8-byte hash for 33 to 64 bytes.
-static uint64 HashLen33to64(const char *s, size_t len) {
- uint64 z = UNALIGNED_LOAD64(s + 24);
- uint64 a = UNALIGNED_LOAD64(s) + (len + UNALIGNED_LOAD64(s + len - 16)) * k0;
- uint64 b = Rotate(a + z, 52);
- uint64 c = Rotate(a, 37);
- a += UNALIGNED_LOAD64(s + 8);
- c += Rotate(a, 7);
- a += UNALIGNED_LOAD64(s + 16);
- uint64 vf = a + z;
- uint64 vs = b + Rotate(a, 31) + c;
- a = UNALIGNED_LOAD64(s + 16) + UNALIGNED_LOAD64(s + len - 32);
- z = UNALIGNED_LOAD64(s + len - 8);
- b = Rotate(a + z, 52);
- c = Rotate(a, 37);
- a += UNALIGNED_LOAD64(s + len - 24);
- c += Rotate(a, 7);
- a += UNALIGNED_LOAD64(s + len - 16);
- uint64 wf = a + z;
- uint64 ws = b + Rotate(a, 31) + c;
- uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0);
- return ShiftMix(r * k0 + vs) * k2;
-}
-
-uint64 CityHash64(const char *s, size_t len) {
- if (len <= 32) {
- if (len <= 16) {
- return HashLen0to16(s, len);
- } else {
- return HashLen17to32(s, len);
- }
- } else if (len <= 64) {
- return HashLen33to64(s, len);
- }
-
- // For strings over 64 bytes we hash the end first, and then as we
- // loop we keep 56 bytes of state: v, w, x, y, and z.
- uint64 x = UNALIGNED_LOAD64(s);
- uint64 y = UNALIGNED_LOAD64(s + len - 16) ^ k1;
- uint64 z = UNALIGNED_LOAD64(s + len - 56) ^ k0;
- pair<uint64, uint64> v = WeakHashLen32WithSeeds(s + len - 64, len, y);
- pair<uint64, uint64> w = WeakHashLen32WithSeeds(s + len - 32, len * k1, k0);
- z += ShiftMix(v.second) * k1;
- x = Rotate(z + x, 39) * k1;
- y = Rotate(y, 33) * k1;
-
- // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
- len = (len - 1) & ~static_cast<size_t>(63);
- do {
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- len -= 64;
- } while (len != 0);
- return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z,
- HashLen16(v.second, w.second) + x);
-}
-
-uint64 CityHash64WithSeed(const char *s, size_t len, uint64 seed) {
- return CityHash64WithSeeds(s, len, k2, seed);
-}
-
-uint64 CityHash64WithSeeds(const char *s, size_t len,
- uint64 seed0, uint64 seed1) {
- return HashLen16(CityHash64(s, len) - seed0, seed1);
-}
-
-// A subroutine for CityHash128(). Returns a decent 128-bit hash for strings
-// of any length representable in ssize_t. Based on City and Murmur.
-static uint128 CityMurmur(const char *s, size_t len, uint128 seed) {
- uint64 a = Uint128Low64(seed);
- uint64 b = Uint128High64(seed);
- uint64 c = 0;
- uint64 d = 0;
- ssize_t l = len - 16;
- if (l <= 0) { // len <= 16
- c = b * k1 + HashLen0to16(s, len);
- d = Rotate(a + (len >= 8 ? UNALIGNED_LOAD64(s) : c), 32);
- } else { // len > 16
- c = HashLen16(UNALIGNED_LOAD64(s + len - 8) + k1, a);
- d = HashLen16(b + len, c + UNALIGNED_LOAD64(s + len - 16));
- a += d;
- do {
- a ^= ShiftMix(UNALIGNED_LOAD64(s) * k1) * k1;
- a *= k1;
- b ^= a;
- c ^= ShiftMix(UNALIGNED_LOAD64(s + 8) * k1) * k1;
- c *= k1;
- d ^= c;
- s += 16;
- l -= 16;
- } while (l > 0);
- }
- a = HashLen16(a, c);
- b = HashLen16(d, b);
- return uint128(a ^ b, HashLen16(b, a));
-}
-
-uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed) {
- if (len < 128) {
- return CityMurmur(s, len, seed);
- }
-
- // We expect len >= 128 to be the common case. Keep 56 bytes of state:
- // v, w, x, y, and z.
- pair<uint64, uint64> v, w;
- uint64 x = Uint128Low64(seed);
- uint64 y = Uint128High64(seed);
- uint64 z = len * k1;
- v.first = Rotate(y ^ k1, 49) * k1 + UNALIGNED_LOAD64(s);
- v.second = Rotate(v.first, 42) * k1 + UNALIGNED_LOAD64(s + 8);
- w.first = Rotate(y + z, 35) * k1 + x;
- w.second = Rotate(x + UNALIGNED_LOAD64(s + 88), 53) * k1;
-
- // This is the same inner loop as CityHash64(), manually unrolled.
- do {
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- len -= 128;
- } while (LIKELY(len >= 128));
- y += Rotate(w.first, 37) * k0 + z;
- x += Rotate(v.first + z, 49) * k0;
- // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
- for (size_t tail_done = 0; tail_done < len; ) {
- tail_done += 32;
- y = Rotate(y - x, 42) * k0 + v.second;
- w.first += UNALIGNED_LOAD64(s + len - tail_done + 16);
- x = Rotate(x, 49) * k0 + w.first;
- w.first += v.first;
- v = WeakHashLen32WithSeeds(s + len - tail_done, v.first, v.second);
- }
- // At this point our 48 bytes of state should contain more than
- // enough information for a strong 128-bit hash. We use two
- // different 48-byte-to-8-byte hashes to get a 16-byte final result.
- x = HashLen16(x, v.first);
- y = HashLen16(y, w.first);
- return uint128(HashLen16(x + v.second, w.second) + y,
- HashLen16(x + w.second, y + v.second));
-}
-
-uint128 CityHash128(const char *s, size_t len) {
- if (len >= 16) {
- return CityHash128WithSeed(s + 16,
- len - 16,
- uint128(UNALIGNED_LOAD64(s) ^ k3,
- UNALIGNED_LOAD64(s + 8)));
- } else if (len >= 8) {
- return CityHash128WithSeed(NULL,
- 0,
- uint128(UNALIGNED_LOAD64(s) ^ (len * k0),
- UNALIGNED_LOAD64(s + len - 8) ^ k1));
- } else {
- return CityHash128WithSeed(s, len, uint128(k0, k1));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
deleted file mode 100644
index 7b290be..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-//
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
-//
-// This file provides a few functions for hashing strings. On x86-64
-// hardware in 2011, CityHash64() is faster than other high-quality
-// hash functions, such as Murmur. This is largely due to higher
-// instruction-level parallelism. CityHash64() and CityHash128() also perform
-// well on hash-quality tests.
-//
-// CityHash128() is optimized for relatively long strings and returns
-// a 128-bit hash. For strings more than about 2000 bytes it can be
-// faster than CityHash64().
-//
-// Functions in the CityHash family are not suitable for cryptography.
-//
-// WARNING: This code has not been tested on big-endian platforms!
-// It is known to work well on little-endian platforms that have a small penalty
-// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs.
-//
-// By the way, for some hash functions, given strings a and b, the hash
-// of a+b is easily derived from the hashes of a and b. This property
-// doesn't hold for any hash functions in this file.
-
-#ifndef CITY_HASH_H_
-#define CITY_HASH_H_
-
-#include <stdlib.h> // for size_t.
-#include <stdint.h>
-#include <utility>
-
-typedef uint8_t uint8;
-typedef uint32_t uint32;
-typedef uint64_t uint64;
-typedef std::pair<uint64, uint64> uint128;
-
-inline uint64 Uint128Low64(const uint128& x) { return x.first; }
-inline uint64 Uint128High64(const uint128& x) { return x.second; }
-
-// Hash function for a byte array.
-uint64 CityHash64(const char *buf, size_t len);
-
-// Hash function for a byte array. For convenience, a 64-bit seed is also
-// hashed into the result.
-uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed);
-
-// Hash function for a byte array. For convenience, two seeds are also
-// hashed into the result.
-uint64 CityHash64WithSeeds(const char *buf, size_t len,
- uint64 seed0, uint64 seed1);
-
-// Hash function for a byte array.
-uint128 CityHash128(const char *s, size_t len);
-
-// Hash function for a byte array. For convenience, a 128-bit seed is also
-// hashed into the result.
-uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed);
-
-// Hash 128 input bits down to 64 bits of output.
-// This is intended to be a reasonably good hash function.
-inline uint64 Hash128to64(const uint128& x) {
- // Murmur-inspired hashing.
- const uint64 kMul = 0x9ddfea08eb382d69ULL;
- uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul;
- a ^= (a >> 47);
- uint64 b = (Uint128High64(x) ^ a) * kMul;
- b ^= (b >> 47);
- b *= kMul;
- return b;
-}
-
-#endif // CITY_HASH_H_
[23/50] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc
warnings. Contributed by Manu Zhang.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 11712ab..60bb6f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -622,7 +622,6 @@ typedef void (*ANY_FUNC_PTR)();
#define DEFINE_NATIVE_LIBRARY(Library) \
static std::map<std::string, NativeTask::ObjectCreatorFunc> Library##ClassMap__; \
extern "C" void * Library##GetFunctionGetter(const std::string & name) { \
- void * ret = NULL; \
std::map<std::string, NativeTask::ObjectCreatorFunc>::iterator itr = Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return (void *)(itr->second); \
@@ -630,7 +629,6 @@ typedef void (*ANY_FUNC_PTR)();
return NULL; \
} \
extern "C" NativeTask::ObjectCreatorFunc Library##GetObjectCreator(const std::string & name) { \
- NativeObject * ret = NULL; \
std::map<std::string, NativeTask::ObjectCreatorFunc>::iterator itr = Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return itr->second; \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
index a59b9d8..59cd8fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
@@ -23,7 +23,7 @@
namespace NativeTask {
BlockCompressStream::BlockCompressStream(OutputStream * stream, uint32_t bufferSizeHint)
- : CompressStream(stream), _compressedBytesWritten(0), _tempBufferSize(0), _tempBuffer(NULL) {
+ : CompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0), _compressedBytesWritten(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
}
@@ -68,7 +68,7 @@ uint64_t BlockCompressStream::compressedBytesWritten() {
//////////////////////////////////////////////////////////////
BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t bufferSizeHint)
- : DecompressStream(stream), _tempBufferSize(0), _tempBuffer(NULL) {
+ : DecompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
_tempDecompressBuffer = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
index bc022a8..15ce2ea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
@@ -103,10 +103,10 @@ protected:
*/
inline void output(const char * buff, uint32_t length) {
while (length > 0) {
- if (length > _out.remain()) {
+ uint32_t remain = _out.remain();
+ if (length > remain) {
flushOutput();
}
- uint32_t remain = _out.remain();
uint32_t cp = length < remain ? length : remain;
simple_memcpy(_out.current(), buff, cp);
buff += cp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
index 40a32df..83da238 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
@@ -24,9 +24,9 @@ const int LENGTH_OF_REFILL_STRING = 6;
const Command CombineHandler::COMBINE(4, "Combine");
CombineHandler::CombineHandler()
- : _combineContext(NULL), _kvIterator(NULL), _writer(NULL), _config(NULL), _kvCached(false),
- _kType(UnknownType), _vType(UnknownType), _combineInputRecordCount(0), _combineInputBytes(0),
- _combineOutputRecordCount(0),_combineOutputBytes(0){
+ : _combineContext(NULL), _kvIterator(NULL), _writer(NULL), _kType(UnknownType),
+ _vType(UnknownType), _config(NULL), _kvCached(false), _combineInputRecordCount(0),
+ _combineInputBytes(0), _combineOutputRecordCount(0), _combineOutputBytes(0) {
}
CombineHandler::~CombineHandler() {
@@ -95,7 +95,6 @@ uint32_t CombineHandler::feedDataToJavaInWritableSerialization() {
* KV: key or value
*/
void CombineHandler::outputKeyOrValue(SerializeInfo & KV, KeyValueType type) {
- uint32_t length = 0;
switch (type) {
case TextType:
output(KV.varBytes, KV.outerLength - KV.buffer.length());
@@ -159,7 +158,6 @@ uint32_t CombineHandler::feedDataToJava(SerializationFramework serializationType
void CombineHandler::handleInput(ByteBuffer & in) {
char * buff = in.current();
uint32_t length = in.remain();
- const char * end = buff + length;
uint32_t remain = length;
char * pos = buff;
if (_asideBuffer.remain() > 0) {
@@ -183,7 +181,7 @@ void CombineHandler::handleInput(ByteBuffer & in) {
THROW_EXCEPTION(IOException, "k/v meta information incomplete");
}
- int kvLength = kvBuffer->lengthConvertEndium();
+ uint32_t kvLength = kvBuffer->lengthConvertEndium();
if (kvLength > remain) {
_asideBytes.resize(kvLength);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
index cc6b5e9..0da23af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
@@ -429,8 +429,8 @@ public:
void rewind(int newPos, int newLimit) {
this->_position = newPos;
- if (newLimit > this->_capacity) {
- THROW_EXCEPTION(IOException, "length larger than input buffer capacity");
+ if (newLimit < 0 || newLimit > this->_capacity) {
+ THROW_EXCEPTION(IOException, "length smaller than zero or larger than input buffer capacity");
}
this->_limit = newLimit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
index ff272ba..d11d823 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
@@ -27,9 +27,9 @@ namespace NativeTask {
///////////////////////////////////////////////////////////
IFileReader::IFileReader(InputStream * stream, SingleSpillInfo * spill, bool deleteInputStream)
- : _deleteSourceStream(deleteInputStream), _stream(stream), _source(NULL),
- _checksumType(spill->checkSumType), _kType(spill->keyType), _vType(spill->valueType),
- _codec(spill->codec), _segmentIndex(-1), _spillInfo(spill), _valuePos(NULL), _valueLen(0) {
+ : _stream(stream), _source(NULL), _checksumType(spill->checkSumType), _kType(spill->keyType),
+ _vType(spill->valueType), _codec(spill->codec), _segmentIndex(-1), _spillInfo(spill),
+ _valuePos(NULL), _valueLen(0), _deleteSourceStream(deleteInputStream) {
_source = new ChecksumInputStream(_stream, _checksumType);
_source->setLimit(0);
_reader.init(128 * 1024, _source, _codec);
@@ -97,9 +97,8 @@ IFileWriter * IFileWriter::create(const std::string & filepath, const MapOutputS
IFileWriter::IFileWriter(OutputStream * stream, ChecksumType checksumType, KeyValueType ktype,
KeyValueType vtype, const string & codec, Counter * counter, bool deleteTargetStream)
- : _deleteTargetStream(deleteTargetStream), _stream(stream), _dest(NULL),
- _checksumType(checksumType), _kType(ktype), _vType(vtype), _codec(codec),
- _recordCounter(counter) {
+ : _stream(stream), _dest(NULL), _checksumType(checksumType), _kType(ktype), _vType(vtype),
+ _codec(codec), _recordCounter(counter), _deleteTargetStream(deleteTargetStream) {
_dest = new ChecksumOutputStream(_stream, _checksumType);
_appendBuffer.init(128 * 1024, _dest, _codec);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
index 34f7072..d2b116d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
@@ -76,9 +76,9 @@ void CombineRunnerWrapper::combine(CombineContext type, KVIterator * kvIterator,
/////////////////////////////////////////////////////////////////
MapOutputCollector::MapOutputCollector(uint32_t numberPartitions, SpillOutputService * spillService)
- : _config(NULL), _buckets(NULL), _keyComparator(NULL), _defaultBlockSize(0),
- _combineRunner(NULL), _spilledRecords(NULL), _spillOutput(spillService), _pool(NULL),
- _numPartitions(numberPartitions) {
+ : _config(NULL), _numPartitions(numberPartitions), _buckets(NULL), _keyComparator(NULL),
+ _combineRunner(NULL), _spilledRecords(NULL), _spillOutput(spillService),
+ _defaultBlockSize(0), _pool(NULL) {
_pool = new MemoryPool();
}
@@ -186,7 +186,7 @@ KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t k
delete spillpath;
}
- dest = dest = partition->allocateKVBuffer(kvlength);
+ dest = partition->allocateKVBuffer(kvlength);
if (NULL == dest) {
// io.sort.mb too small, cann't proceed
// should not get here, cause get_buffer_to_put can throw OOM exception
@@ -294,7 +294,7 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime;
LOG(
- "[MapOutputCollector::mid_spill] Sort and spill: {spilled file path: %s, id: %d, collect: %llu ms, sort: %llu ms, spill: %llu ms, records: %llu, uncompressed total bytes: %llu, compressed total bytes: %llu}",
+ "[MapOutputCollector::mid_spill] Sort and spill: {spilled file path: %s, id: %d, collect: %"PRIu64" ms, sort: %"PRIu64" ms, spill: %"PRIu64" ms, records: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64"}",
info->path.c_str(), _spillInfos.getSpillCount(), collecttime / M, metrics.sortTime / M, spillTime / M,
metrics.recordCount, info->getEndPosition(), info->getRealEndPosition());
@@ -339,7 +339,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
SortMetrics metrics;
sortPartitions(_spec.sortOrder, _spec.sortAlgorithm, NULL, metrics);
- LOG("[MapOutputCollector::mid_spill] Sort final in memory kvs: {sort: %llu ms, records: %llu}",
+ LOG("[MapOutputCollector::mid_spill] Sort final in memory kvs: {sort: %"PRIu64" ms, records: %"PRIu64"}",
metrics.sortTime / M, metrics.recordCount);
merger->addMergeEntry(new MemoryMergeEntry(_buckets, _numPartitions));
@@ -347,7 +347,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
Timer timer;
merger->merge();
LOG(
- "[MapOutputCollector::final_merge_and_spill] Merge and Spill:{spilled file id: %d, merge and spill time: %llu ms}",
+ "[MapOutputCollector::final_merge_and_spill] Merge and Spill:{spilled file id: %d, merge and spill time: %"PRIu64" ms}",
_spillInfos.getSpillCount(), (timer.now() - timer.last()) / M);
delete merger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
index 0692a33..b7150ac 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
@@ -55,8 +55,8 @@ private:
public:
CombineRunnerWrapper(Config * config, SpillOutputService * service)
- : _spillOutput(service), _config(config), _isJavaCombiner(false), _combineRunner(NULL),
- _combinerInited(false) {
+ : _config(config), _combineRunner(NULL), _isJavaCombiner(false),
+ _combinerInited(false), _spillOutput(service) {
}
~CombineRunnerWrapper() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
index a8d7e1b..5434980 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
@@ -162,7 +162,7 @@ void Merger::merge() {
_writer->getStatistics(output_size, real_output_size);
if (total_record != 0) {
- LOG("[Merge] Merged segment#: %lu, record#: %llu, avg record size: %llu, uncompressed total bytes: %llu, compressed total bytes: %llu, time: %llu ms",
+ LOG("[Merge] Merged segment#: %lu, record#: %"PRIu64", avg record size: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
_entries.size(),
total_record,
output_size / (total_record),
@@ -170,7 +170,7 @@ void Merger::merge() {
real_output_size,
interval / M);
} else {
- LOG("[Merge] Merged segments#, %lu, uncompressed total bytes: %llu, compressed total bytes: %llu, time: %llu ms",
+ LOG("[Merge] Merged segments#, %lu, uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
_entries.size(),
output_size,
real_output_size,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
index 37c45af..cf5e33b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
@@ -50,8 +50,8 @@ private:
public:
PartitionBucket(MemoryPool * pool, uint32_t partition, ComparatorPtr comparator,
ICombineRunner * combineRunner, uint32_t blockSize)
- : _pool(pool), _partition(partition), _keyComparator(comparator),
- _combineRunner(combineRunner), _blockSize(blockSize), _sorted(false) {
+ : _pool(pool), _partition(partition), _blockSize(blockSize),
+ _keyComparator(comparator), _combineRunner(combineRunner), _sorted(false) {
if (NULL == _pool || NULL == comparator) {
THROW_EXCEPTION_EX(IOException, "pool is NULL, or comparator is not set");
}
@@ -102,11 +102,11 @@ public:
}
_sorted = false;
MemoryBlock * memBlock = NULL;
- uint32_t memBockSize = _memBlocks.size();
- if (memBockSize > 0) {
- memBlock = _memBlocks[memBockSize - 1];
+ uint32_t memBlockSize = _memBlocks.size();
+ if (memBlockSize > 0) {
+ memBlock = _memBlocks[memBlockSize - 1];
}
- if (NULL != memBockSize && memBlock->remainSpace() >= kvLength) {
+ if (NULL != memBlock && memBlock->remainSpace() >= kvLength) {
return memBlock->allocateKVBuffer(kvLength);
} else {
uint32_t min = kvLength;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
index 89770ad..4212533 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
@@ -30,6 +30,9 @@
#include <memory.h>
#include <fcntl.h>
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
#include <limits>
#include <string>
#include <vector>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
index 4e5b94a..7baaa0c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
@@ -36,19 +36,19 @@ string StringUtil::ToString(uint32_t v) {
string StringUtil::ToString(int64_t v) {
char tmp[32];
- snprintf(tmp, 32, "%lld", (long long int)v);
+ snprintf(tmp, 32, "%"PRId64, v);
return tmp;
}
string StringUtil::ToString(int64_t v, char pad, int64_t len) {
char tmp[32];
- snprintf(tmp, 32, "%%%c%lldlld", pad, len);
+ snprintf(tmp, 32, "%%%c%"PRId64""PRId64, pad, len);
return Format(tmp, v);
}
string StringUtil::ToString(uint64_t v) {
char tmp[32];
- snprintf(tmp, 32, "%llu", (long long unsigned int)v);
+ snprintf(tmp, 32, "%"PRIu64, v);
return tmp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
index de02ac7..1b99286 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
@@ -181,13 +181,11 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl
char * dest = new char[blockSize + 8];
CompressResult result;
Timer t;
- int compressedSize;
for (size_t start = 0; start < data.length(); start += blockSize) {
size_t currentblocksize = std::min(data.length() - start, blockSize);
uint64_t startTime = t.now();
for (int i = 0; i < times; i++) {
int osize = LZ4_compress((char*)data.data() + start, outputBuffer, currentblocksize);
- compressedSize = osize;
result.compressedSize += osize;
result.uncompressedSize += currentblocksize;
}
@@ -197,6 +195,7 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl
for (int i = 0; i < times; i++) {
// memset(dest, 0, currentblocksize+8);
int osize = LZ4_uncompress(outputBuffer, dest, currentblocksize);
+ ASSERT_EQ(currentblocksize, osize);
// printf("%016llx blocksize: %lu\n", bswap64(*(uint64_t*)(dest+currentblocksize)), currentblocksize);
}
endTime = t.now();
@@ -239,7 +238,7 @@ void MeasureSingleFileSnappy(const string & path, CompressResult & total, size_t
char * dest = new char[blockSize];
CompressResult result;
Timer t;
- int compressedSize;
+ int compressedSize = -1;
for (size_t start = 0; start < data.length(); start += blockSize) {
size_t currentblocksize = std::min(data.length() - start, blockSize);
uint64_t startTime = t.now();
@@ -272,7 +271,7 @@ TEST(Perf, RawCompressionSnappy) {
vector<FileEntry> inputfiles;
FileSystem::getLocal().list(inputdir, inputfiles);
CompressResult total;
- printf("Block size: %lldK\n", blockSize / 1024);
+ printf("Block size: %"PRId64"K\n", blockSize / 1024);
for (size_t i = 0; i < inputfiles.size(); i++) {
if (!inputfiles[i].isDirectory) {
MeasureSingleFileSnappy((inputdir + "/" + inputfiles[i].name).c_str(), total, blockSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 1d8b520..955ef29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -119,13 +119,17 @@ void TestIFileWriteRead2(vector<pair<string, string> > & kvs, char * buff, size_
InputBuffer inputBuffer = InputBuffer(buff, outputBuffer.tell());
IFileReader * ir = new IFileReader(&inputBuffer, info);
timer.reset();
+ int sum = 0;
while (ir->nextPartition()) {
const char * key, *value;
uint32_t keyLen, valueLen;
while (NULL != (key = ir->nextKey(keyLen))) {
value = ir->value(valueLen);
+ sum += value[0];
}
}
+ // use the result so that value() calls don't get optimized out
+ ASSERT_NE(0xdeadbeef, sum);
LOG("%s",
timer.getSpeedM2(" Read data", info->getEndPosition(), info->getRealEndPosition()).c_str());
delete ir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
index 228cace..c03f02e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
@@ -35,7 +35,7 @@ TEST(ByteBuffer, read) {
ASSERT_EQ(buff, byteBuffer.current());
ASSERT_EQ(0, byteBuffer.remain());
- int newPos = byteBuffer.advance(3);
+ byteBuffer.advance(3);
ASSERT_EQ(3, byteBuffer.current() - byteBuffer.base());
byteBuffer.rewind(10, 20);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
index cd8fb27..a2dd909 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
@@ -43,7 +43,7 @@ TEST(FixSizeContainer, test) {
container->fill(toBeFilled.c_str(), toBeFilled.length());
- for (int i = 0; i < container->position(); i++) {
+ for (uint32_t i = 0; i < container->position(); i++) {
char * c = container->base() + i;
ASSERT_EQ(toBeFilled[i], *c);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
index 6e79796..88a3258 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
@@ -88,8 +88,6 @@ void TestKeyGroupIterator() {
int * keyPtr = (int *)key;
const char * value = NULL;
while (NULL != (value = groupIterator->nextValue(length))) {
- int * valuePtr = (int *)value;
-
if (actualKeyCount.find(*keyPtr) == actualKeyCount.end()) {
actualKeyCount[*keyPtr] = 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
index 87fba15..c11d5ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
@@ -27,13 +27,12 @@ namespace NativeTask {
TEST(MemoryBlockIterator, test) {
const uint32_t BUFFER_LENGTH = 100;
- const uint32_t BLOCK_ID = 3;
char * bytes = new char[BUFFER_LENGTH];
MemoryBlock block(bytes, BUFFER_LENGTH);
const uint32_t KV_SIZE = 60;
- KVBuffer * kv1 = block.allocateKVBuffer(KV_SIZE);
- KVBuffer * kv2 = block.allocateKVBuffer(KV_SIZE);
+ block.allocateKVBuffer(KV_SIZE);
+ block.allocateKVBuffer(KV_SIZE);
MemBlockIterator iter(&block);
@@ -50,7 +49,6 @@ class MemoryBlockFactory {
public:
static MemoryBlock * create(std::vector<int> & keys) {
const uint32_t BUFFER_LENGTH = 1000;
- const uint32_t BLOCK_ID = 3;
char * bytes = new char[BUFFER_LENGTH];
MemoryBlock * block1 = new MemoryBlock(bytes, BUFFER_LENGTH);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
index dc60ac3..1d48c79 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
@@ -44,6 +44,7 @@ TEST(MemoryBlock, test) {
ASSERT_EQ(2, block.getKVCount());
ASSERT_EQ(kv1, block.getKVBuffer(0));
+ ASSERT_EQ(kv2, block.getKVBuffer(1));
ASSERT_EQ(BUFFER_LENGTH - 2 * KV_SIZE, block.remainSpace());
ASSERT_EQ(false, block.sorted());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
index e2f854c..501e005 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
@@ -24,7 +24,7 @@ static uint64_t test_length(int64_t len, size_t size, size_t loopTime) {
TestConfig.setInt(GenerateLen, len);
Generate(data, size, "bytes");
Timer t;
- uint64_t ret;
+ uint64_t ret = 0;
for (size_t m = 0; m < loopTime; m++) {
for (size_t i = 0; i < data.size(); i++) {
ret += Hash::BytesHash(data[i].c_str(), data[i].length());
[14/50] [abbrv] git commit: MAPREDUCE-5995. native-task: Revert
changes to Text internals. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-5995. native-task: Revert changes to Text internals. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613828 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78d86a98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78d86a98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78d86a98
Branch: refs/heads/HDFS-6581
Commit: 78d86a983960f231f51e47579c0b1ae73f9645fd
Parents: 43917e5
Author: Todd Lipcon <to...@apache.org>
Authored: Sun Jul 27 19:04:14 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Sun Jul 27 19:04:14 2014 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/io/Text.java | 6 +-----
hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/serde/TextSerializer.java | 9 +--------
3 files changed, 3 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index 73cd1e6..3dc5076 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -226,10 +226,6 @@ public class Text extends BinaryComparable
this.length = len;
}
- public void setLength(int len) {
- this.length = len;
- }
-
/**
* Append a range of bytes to the end of the given text
* @param utf8 the data to copy from
@@ -264,7 +260,7 @@ public class Text extends BinaryComparable
* @param len the number of bytes we need
* @param keepData should the old data be kept
*/
- public void setCapacity(int len, boolean keepData) {
+ private void setCapacity(int len, boolean keepData) {
if (bytes == null || bytes.length < len) {
if (bytes != null && keepData) {
bytes = Arrays.copyOf(bytes, Math.max(len,length << 1));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 92c94a8..42e0af3 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -7,3 +7,4 @@ MAPREDUCE-5996. native-task: Rename system tests into standard directory layout
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
+MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
index 63a64de..cde4c7b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
@@ -42,13 +42,6 @@ public class TextSerializer implements INativeSerializer<Text>, INativeComparabl
@Override
public void deserialize(DataInput in, int length, Text w) throws IOException {
- try {
- w.setCapacity(length, true);
- w.setLength(length);
- } catch (final Exception e) {
- throw new IOException(e);
- }
- final byte[] bytes = w.getBytes();
- in.readFully(bytes, 0, length);
+ w.readWithKnownLength(in, length);
}
}
[44/50] [abbrv] git commit: HADOOP-11088. Unittest TestKeyShell,
TestCredShell and TestKMS assume UNIX path separator for JECKS key
store path. Contributed by Xiaoyu Yao.
Posted by ar...@apache.org.
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX path separator for JECKS key store path. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/957414d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/957414d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/957414d4
Branch: refs/heads/HDFS-6581
Commit: 957414d4cb57cb8172070cc53530b7da78b8c9ca
Parents: 54e5794
Author: cnauroth <cn...@apache.org>
Authored: Fri Sep 12 14:50:07 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Fri Sep 12 14:50:07 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/crypto/key/TestKeyShell.java | 4 +++-
.../hadoop/security/alias/TestCredShell.java | 22 +++++++++++++-------
.../crypto/key/kms/server/KMSConfiguration.java | 15 ++++++-------
.../hadoop/crypto/key/kms/server/MiniKMS.java | 3 ++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 3 ++-
6 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1176a21..051eac1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -793,6 +793,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is
case sensitive. (tucu)
+ HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
+ path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
index d65d9b2..3407eb7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
@@ -24,6 +24,7 @@ import java.io.PrintStream;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +52,8 @@ public class TestKeyShell {
if (!tmpDir.mkdirs()) {
throw new IOException("Unable to create " + tmpDir);
}
- jceksProvider = "jceks://file" + tmpDir + "/keystore.jceks";
+ final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
+ jceksProvider = "jceks://file" + jksPath.toUri();
initialStdOut = System.out;
initialStdErr = System.err;
System.setOut(new PrintStream(outContent));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index b9f0dc9..c890362 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;
@@ -37,18 +38,23 @@ public class TestCredShell {
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private static final File tmpDir =
new File(System.getProperty("test.build.data", "/tmp"), "creds");
-
+
+ /* The default JCEKS provider - for testing purposes */
+ private String jceksProvider;
+
@Before
public void setup() throws Exception {
System.setOut(new PrintStream(outContent));
System.setErr(new PrintStream(errContent));
+ final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
+ jceksProvider = "jceks://file" + jksPath.toUri();
}
@Test
public void testCredentialSuccessfulLifecycle() throws Exception {
outContent.reset();
String[] args1 = {"create", "credential1", "-value", "p@ssw0rd", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
int rc = 0;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
@@ -59,14 +65,14 @@ public class TestCredShell {
outContent.reset();
String[] args2 = {"list", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
rc = cs.run(args2);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("credential1"));
outContent.reset();
String[] args4 = {"delete", "credential1", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
rc = cs.run(args4);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("credential1 has been successfully " +
@@ -74,7 +80,7 @@ public class TestCredShell {
outContent.reset();
String[] args5 = {"list", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
rc = cs.run(args5);
assertEquals(0, rc);
assertFalse(outContent.toString(), outContent.toString().contains("credential1"));
@@ -132,7 +138,7 @@ public class TestCredShell {
@Test
public void testPromptForCredentialWithEmptyPasswd() throws Exception {
String[] args1 = {"create", "credential1", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
ArrayList<String> passwords = new ArrayList<String>();
passwords.add(null);
passwords.add("p@ssw0rd");
@@ -148,7 +154,7 @@ public class TestCredShell {
@Test
public void testPromptForCredential() throws Exception {
String[] args1 = {"create", "credential1", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
ArrayList<String> passwords = new ArrayList<String>();
passwords.add("p@ssw0rd");
passwords.add("p@ssw0rd");
@@ -162,7 +168,7 @@ public class TestCredShell {
"created."));
String[] args2 = {"delete", "credential1", "-provider",
- "jceks://file" + tmpDir + "/credstore.jceks"};
+ jceksProvider};
rc = shell.run(args2);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("credential1 has been successfully " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index a7daa24..f028119 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import java.io.File;
import java.net.MalformedURLException;
@@ -72,15 +73,13 @@ public class KMSConfiguration {
String confDir = System.getProperty(KMS_CONFIG_DIR);
if (confDir != null) {
try {
- if (!confDir.startsWith("/")) {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
"' must be an absolute path: " + confDir);
}
- if (!confDir.endsWith("/")) {
- confDir += "/";
- }
for (String resource : resources) {
- conf.addResource(new URL("file://" + confDir + resource));
+ conf.addResource(new URL("file://" + new Path(confDir, resource).toUri()));
}
} catch (MalformedURLException ex) {
throw new RuntimeException(ex);
@@ -105,13 +104,11 @@ public class KMSConfiguration {
boolean newer = false;
String confDir = System.getProperty(KMS_CONFIG_DIR);
if (confDir != null) {
- if (!confDir.startsWith("/")) {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
"' must be an absolute path: " + confDir);
}
- if (!confDir.endsWith("/")) {
- confDir += "/";
- }
File f = new File(confDir, KMS_ACLS_XML);
// at least 100ms newer than time, we do this to ensure the file
// has been properly closed/flushed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 5a6d4c5..195eee8 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.security.SslSocketConnector;
@@ -158,7 +159,7 @@ public class MiniKMS {
if (!kmsFile.exists()) {
Configuration kms = new Configuration(false);
kms.set("hadoop.security.key.provider.path",
- "jceks://file@" + kmsConfDir + "/kms.keystore");
+ "jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
kms.set("hadoop.kms.authentication.type", "simple");
Writer writer = new FileWriter(kmsFile);
kms.writeXml(writer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 1ca0c0d..f4f9fea 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
@@ -117,7 +118,7 @@ public class TestKMS {
protected Configuration createBaseKMSConf(File keyStoreDir) throws Exception {
Configuration conf = new Configuration(false);
conf.set("hadoop.security.key.provider.path",
- "jceks://file@/" + keyStoreDir.getAbsolutePath() + "/kms.keystore");
+ "jceks://file@" + new Path(keyStoreDir.getAbsolutePath(), "kms.keystore").toUri());
conf.set("hadoop.kms.authentication.type", "simple");
return conf;
}
[05/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into
standard directory layout. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
deleted file mode 100644
index 8a4aa6f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CombinerTest {
- private FileSystem fs;
- private String inputpath;
- private String nativeoutputpath;
- private String hadoopoutputpath;
-
- @Test
- public void testWordCountCombiner() {
- try {
-
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
-
- final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
- commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
- final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
-
- nativejob.waitForCompletion(true);
-
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- normaljob.waitForCompletion(true);
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
- assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
- nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
- }
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- conf.addcombinerConf();
-
- this.fs = FileSystem.get(conf);
-
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
- if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(
- conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000),
- Text.class.getName(),
- Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
- }
-
- this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
- this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
- }
-
- protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
- throws Exception {
- final Configuration conf = new Configuration(inputConf);
- conf.set("fileoutputpath", outputpath);
- final FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputpath))) {
- fs.delete(new Path(outputpath));
- }
- fs.close();
- final Job job = new Job(conf, jobname);
- job.setJarByClass(WordCount.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- return job;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
deleted file mode 100644
index 50953e0..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Test;
-
-public class LargeKVCombinerTest {
-
- @Test
- public void testLargeValueCombiner(){
- final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
- deafult_KVSize_Maximum);
- final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
- final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
- final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
- try {
- final FileSystem fs = FileSystem.get(normalConf);
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
-
- int max = i;
- int min = Math.max(i / 4, max - 10);
-
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max);
-
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- fs.delete(new Path(inputPath), true);
- new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
- 1000000), IntWritable.class.getName(),
- Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
-
- final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
- final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
-
- nativejob.waitForCompletion(true);
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- normaljob.waitForCompletion(true);
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-
- final String reason = "LargeKVCombinerTest failed with, min size: " + min
- + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
-
- assertEquals(reason, true, compareRet);
-// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
-// nativeReduceGroups.getValue(), normalReduceGroups.getValue());
- }
- fs.close();
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
deleted file mode 100644
index 5691e02..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.junit.Before;
-import org.junit.Test;
-
-public class OldAPICombinerTest {
- private FileSystem fs;
- private String inputpath;
-
- @Test
- public void testWordCountCombinerWithOldAPI() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
- final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
- RunningJob nativeRunning = JobClient.runJob(nativeJob);
-
- Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
- final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
-
- RunningJob normalRunning = JobClient.runJob(normalJob);
- Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-
- assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- conf.addcombinerConf();
- this.fs = FileSystem.get(conf);
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
- if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
- Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
- }
- }
-
- private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
- throws Exception {
- final JobConf jobConf = new JobConf(configuration);
- final FileSystem fs = FileSystem.get(configuration);
- if (fs.exists(new Path(output))) {
- fs.delete(new Path(output), true);
- }
- fs.close();
- jobConf.setJobName(name);
- jobConf.setOutputKeyClass(Text.class);
- jobConf.setOutputValueClass(IntWritable.class);
- jobConf.setMapperClass(WordCountWithOldAPI.TokenizerMapperWithOldAPI.class);
- jobConf.setCombinerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
- jobConf.setReducerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
-
- jobConf.setInputFormat(SequenceFileInputFormat.class);
- jobConf.setOutputFormat(TextOutputFormat.class);
-
- FileInputFormat.setInputPaths(jobConf, new Path(input));
- FileOutputFormat.setOutputPath(jobConf, new Path(output));
- return jobConf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
deleted file mode 100644
index 490b82b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-
-public class WordCount {
-
- private static Log LOG = LogFactory.getLog(WordCount.class);
-
- public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
-
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
- final StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- private final IntWritable result = new IntWritable();
-
- @Override
- public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
- InterruptedException {
- int sum = 0;
- for (final IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(key, result);
- }
- }
-
- public static void main(String[] args) throws Exception {
- final Configuration conf = new Configuration();
- final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length != 2) {
- System.err.println("Usage: wordcount <in> <out>");
- System.exit(2);
- }
- final Job job = new Job(conf, conf.get(MRJobConfig.JOB_NAME, "word count"));
- job.setJarByClass(WordCount.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
- System.exit(job.waitForCompletion(true) ? 0 : 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
deleted file mode 100644
index a11ea91..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-public class WordCountWithOldAPI {
-
- public static class TokenizerMapperWithOldAPI extends MapReduceBase implements
- Mapper<Object, Text, Text, IntWritable> {
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
- throws IOException {
- final StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- output.collect(word, one);
- }
- }
- }
-
- public static class IntSumReducerWithOldAPI extends MapReduceBase implements
- Reducer<Text, IntWritable, Text, IntWritable> {
- private final IntWritable result = new IntWritable();
-
- @Override
- public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- int sum = 0;
- while (values.hasNext()) {
- sum += values.next().get();
- }
- result.set(sum);
- output.collect(key, result);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
deleted file mode 100644
index 83c4794..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class CompressMapper {
- public static final String inputFile = "./compress/input.txt";
- public static final String outputFileDir = "./compress/output/";
-
- public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
-
- @Override
- protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static Job getCompressJob(String jobname, Configuration conf) {
- Job job = null;
- try {
- job = new Job(conf, jobname + "-CompressMapperJob");
- job.setJarByClass(CompressMapper.class);
- job.setMapperClass(TextCompressMapper.class);
- job.setOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Text.class);
- final Path outputpath = new Path(outputFileDir + jobname);
- // if output file exists ,delete it
- final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
- if (hdfs.exists(outputpath)) {
- hdfs.delete(outputpath);
- }
- hdfs.close();
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputFile));
- FileOutputFormat.setOutputPath(job, outputpath);
- } catch (final Exception e) {
- e.printStackTrace();
- }
- return job;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
deleted file mode 100644
index 0406375..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CompressTest {
-
- @Test
- public void testSnappyCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
- job.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
- CompressMapper.outputFileDir + "hadoopsnappy");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testGzipCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativegzip", conf);
- job.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
- CompressMapper.outputFileDir + "hadoopgzip");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testBzip2Compress() throws Exception {
- final Configuration nativeconf = ScenarioConfiguration.getNativeConfiguration();
- nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job nativejob = CompressMapper.getCompressJob("nativebzip2", nativeconf);
- nativejob.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
- CompressMapper.outputFileDir + "hadoopbzip2");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testLz4Compress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
- nativeJob.waitForCompletion(true);
-
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
- hadoopJob.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
- CompressMapper.outputFileDir + "hadooplz4");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testDefaultCompress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativedefault", nativeConf);
- nativeJob.waitForCompletion(true);
-
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadoopdefault", hadoopConf);
- hadoopJob.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativedefault",
- CompressMapper.outputFileDir + "hadoopdefault");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- final FileSystem fs = FileSystem.get(conf);
- final Path path = new Path(CompressMapper.inputFile);
- fs.delete(path);
- if (!fs.exists(path)) {
- new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
- TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
- Text.class.getName(), Text.class.getName(), conf)
- .createSequenceTestFile(CompressMapper.inputFile);
-
- }
- fs.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
deleted file mode 100644
index ce23780..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, IntWritable> {
-
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(os);
-
- @Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
- int hashSum = 0;
- for (final VTYPE val : values) {
- if (val instanceof Writable) {
- os.reset();
- ((Writable) val).write(dos);
- final int hash = Arrays.hashCode(os.toByteArray());
- hashSum += hash;
- }
- }
-
- context.write(key, new IntWritable(hashSum));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
deleted file mode 100644
index 6d683f8..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class KVJob {
- public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
- public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
- Job job = null;
-
- public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
- @Override
- public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
- public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
-
- @Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
- long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
- final CRC32 crc32 = new CRC32();
- for (final VTYPE val : values) {
- crc32.reset();
- crc32.update(BytesFactory.toBytes(val));
- resultlong += crc32.getValue();
- }
- final VTYPE V = null;
- context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
- }
- }
-
- public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath,
- String outputpath) throws Exception {
- job = new Job(conf, jobname);
- job.setJarByClass(KVJob.class);
- job.setMapperClass(KVJob.ValueMapper.class);
- job.setOutputKeyClass(keyclass);
- job.setMapOutputValueClass(valueclass);
-
- if (conf.get(TestConstants.NATIVETASK_KVTEST_CREATEFILE).equals("true")) {
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(inputpath), true);
- fs.close();
- final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
- TestConstants.FILESIZE_KEY, "1000")),
- keyclass.getName(), valueclass.getName(), conf);
- testfile.createSequenceTestFile(inputpath);
-
- }
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- }
-
- public void runJob() throws Exception {
-
- job.waitForCompletion(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
deleted file mode 100644
index 1e08854..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class KVTest {
- private static Class<?>[] keyclasses = null;
- private static Class<?>[] valueclasses = null;
- private static String[] keyclassNames = null;
- private static String[] valueclassNames = null;
-
- private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
- private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();
- static {
- nativekvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
- hadoopkvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
- }
-
- @Parameters(name = "key:{0}\nvalue:{1}")
- public static Iterable<Class<?>[]> data() {
- final String valueclassesStr = nativekvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
- System.out.println(valueclassesStr);
- valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < valueclassNames.length; i++) {
- try {
- if (valueclassNames[i].equals("")) {
- continue;
- }
- tmpvalueclasses.add(Class.forName(valueclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
- }
- }
- valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
- final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
- System.out.println(keyclassesStr);
- keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < keyclassNames.length; i++) {
- try {
- if (keyclassNames[i].equals("")) {
- continue;
- }
- tmpkeyclasses.add(Class.forName(keyclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
- }
- }
- keyclasses = tmpkeyclasses.toArray(new Class[tmpkeyclasses.size()]);
- final Class<?>[][] kvgroup = new Class<?>[keyclassNames.length * valueclassNames.length][2];
- for (int i = 0; i < keyclassNames.length; i++) {
- final int tmpindex = i * valueclassNames.length;
- for (int j = 0; j < valueclassNames.length; j++) {
- kvgroup[tmpindex + j][0] = keyclasses[i];
- kvgroup[tmpindex + j][1] = valueclasses[j];
- }
- }
- return Arrays.asList(kvgroup);
- }
-
- private final Class<?> keyclass;
- private final Class<?> valueclass;
-
- public KVTest(Class<?> keyclass, Class<?> valueclass) {
- this.keyclass = keyclass;
- this.valueclass = valueclass;
-
- }
-
- @Test
- public void testKVCompability() {
-
- try {
- final String nativeoutput = this.runNativeTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final String normaloutput = this.runNormalTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
- final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- if(compareRet){
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(nativeoutput), true);
- fs.delete(new Path(normaloutput), true);
- fs.delete(new Path(input), true);
- fs.close();
- }
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- } catch (final IOException e) {
- assertEquals("test run exception:", null, e);
- } catch (final Exception e) {
- assertEquals("test run exception:", null, e);
- }
- }
-
- @Before
- public void startUp() {
-
- }
-
- private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(nativekvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- try {
- final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "native testcase run time error.";
- }
- return outputpath;
- }
-
- private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = hadoopkvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
- + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- try {
- final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
deleted file mode 100644
index 900b058..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Test;
-
-public class LargeKVTest {
-
- @Test
- public void testKeySize() {
- runKVSizeTests(Text.class, IntWritable.class);
- }
-
- @Test
- public void testValueSize() {
- runKVSizeTests(IntWritable.class, Text.class);
- }
-
- private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- static {
- nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
- nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
- normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- }
-
- public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
- if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
- return;
- }
- final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
- deafult_KVSize_Maximum);
- try {
-
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
- int min = i / 4;
- int max = i;
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
- + keyClass.getName() + ", valueClass: " + valueClass.getName());
-
- final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
- valueClass, nativeConf);
- final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
- valueClass, normalConf);
- final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
- final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
- + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
- + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
- assertEquals(reason, true, compareRet);
- }
- } catch (final Exception e) {
- // TODO: handle exception
- // assertEquals("test run exception:", null, e);
- e.printStackTrace();
- }
- }
-
- private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws Exception {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
- fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-
- private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws IOException {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
- fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
deleted file mode 100644
index a194697..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-
-
-public class TestInputFile {
-
- public static class KVSizeScope {
- private static final int DefaultMinNum = 1;
- private static final int DefaultMaxNum = 64;
-
- public int minBytesNum;
- public int maxBytesNum;
-
- public KVSizeScope() {
- this.minBytesNum = DefaultMinNum;
- this.maxBytesNum = DefaultMaxNum;
- }
-
- public KVSizeScope(int min, int max) {
- this.minBytesNum = min;
- this.maxBytesNum = max;
- }
- }
-
- private static HashMap<String, KVSizeScope> map = new HashMap<String, KVSizeScope>();
-
- private byte[] databuf = null;
- private final String keyClsName, valueClsName;
- private int filesize = 0;
- private int keyMaxBytesNum, keyMinBytesNum;
- private int valueMaxBytesNum, valueMinBytesNum;
- private SequenceFile.Writer writer = null;
- Random r = new Random();
- public static final int DATABUFSIZE = 1 << 22; // 4M
-
- private enum State {
- KEY, VALUE
- };
-
- static {
- map.put(BooleanWritable.class.getName(), new KVSizeScope(1, 1));
- map.put(DoubleWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(FloatWritable.class.getName(), new KVSizeScope(4, 4));
- map.put(VLongWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(ByteWritable.class.getName(), new KVSizeScope(1, 1));
- map.put(LongWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(VIntWritable.class.getName(), new KVSizeScope(4, 4));
- map.put(IntWritable.class.getName(), new KVSizeScope(4, 4));
- }
-
- public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception {
- this.filesize = filesize;
- this.databuf = new byte[DATABUFSIZE];
- this.keyClsName = keytype;
- this.valueClsName = valuetype;
- final int defaultMinBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MIN, 1);
- final int defaultMaxBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX, 64);
-
- if (map.get(keytype) != null) {
- keyMinBytesNum = map.get(keytype).minBytesNum;
- keyMaxBytesNum = map.get(keytype).maxBytesNum;
- } else {
- keyMinBytesNum = defaultMinBytes;
- keyMaxBytesNum = defaultMaxBytes;
- }
-
- if (map.get(valuetype) != null) {
- valueMinBytesNum = map.get(valuetype).minBytesNum;
- valueMaxBytesNum = map.get(valuetype).maxBytesNum;
- } else {
- valueMinBytesNum = defaultMinBytes;
- valueMaxBytesNum = defaultMaxBytes;
- }
- }
-
- public void createSequenceTestFile(String filepath) throws Exception {
- int FULL_BYTE_SPACE = 256;
- createSequenceTestFile(filepath, FULL_BYTE_SPACE);
- }
-
- public void createSequenceTestFile(String filepath, int base) throws Exception {
- createSequenceTestFile(filepath, base, (byte)0);
- }
-
- public void createSequenceTestFile(String filepath, int base, byte start) throws Exception {
- System.out.println("create file " + filepath);
- System.out.println(keyClsName + " " + valueClsName);
- Class<?> tmpkeycls, tmpvaluecls;
- try {
- tmpkeycls = Class.forName(keyClsName);
- } catch (final ClassNotFoundException e) {
- throw new Exception("key class not found: ", e);
- }
- try {
- tmpvaluecls = Class.forName(valueClsName);
- } catch (final ClassNotFoundException e) {
- throw new Exception("key class not found: ", e);
- }
- try {
- final Path outputfilepath = new Path(filepath);
- final ScenarioConfiguration conf= new ScenarioConfiguration();
- final FileSystem hdfs = outputfilepath.getFileSystem(conf);
- writer = new SequenceFile.Writer(hdfs, conf, outputfilepath, tmpkeycls, tmpvaluecls);
- } catch (final Exception e) {
- e.printStackTrace();
- }
-
- int tmpfilesize = this.filesize;
- while (tmpfilesize > DATABUFSIZE) {
- nextRandomBytes(databuf, base, start);
- final int size = flushBuf(DATABUFSIZE);
- tmpfilesize -= size;
- }
- nextRandomBytes(databuf, base, start);
- flushBuf(tmpfilesize);
-
- if (writer != null) {
- IOUtils.closeStream(writer);
- } else {
- throw new Exception("no writer to create sequenceTestFile!");
- }
- }
-
- private void nextRandomBytes(byte[] buf, int base) {
- nextRandomBytes(buf, base, (byte)0);
- }
-
- private void nextRandomBytes(byte[] buf, int base, byte start) {
- r.nextBytes(buf);
- for (int i = 0; i < buf.length; i++) {
- buf[i] = (byte) ((buf[i] & 0xFF) % base + start);
- }
- }
-
- private int flushBuf(int buflen) throws Exception {
- final Random r = new Random();
- int keybytesnum = 0;
- int valuebytesnum = 0;
- int offset = 0;
-
- while (offset < buflen) {
- final int remains = buflen - offset;
- keybytesnum = keyMaxBytesNum;
- if (keyMaxBytesNum != keyMinBytesNum) {
- keybytesnum = keyMinBytesNum + r.nextInt(keyMaxBytesNum - keyMinBytesNum);
- }
-
- valuebytesnum = valueMaxBytesNum;
- if (valueMaxBytesNum != valueMinBytesNum) {
- valuebytesnum = valueMinBytesNum + r.nextInt(valueMaxBytesNum - valueMinBytesNum);
- }
-
- if (keybytesnum + valuebytesnum > remains) {
- break;
- }
-
- final byte[] key = new byte[keybytesnum];
- final byte[] value = new byte[valuebytesnum];
-
- System.arraycopy(databuf, offset, key, 0, keybytesnum);
- offset += keybytesnum;
-
- System.arraycopy(databuf, offset, value, 0, valuebytesnum);
- offset += valuebytesnum;
-
- try {
- writer.append(BytesFactory.newObject(key, this.keyClsName), BytesFactory.newObject(value, this.valueClsName));
- } catch (final IOException e) {
- e.printStackTrace();
- throw new Exception("sequence file create failed", e);
- }
- }
- return offset;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
deleted file mode 100644
index b7b03e7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class NonSortTest {
-
- @Test
- public void nonSortTest() throws Exception {
- Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
- nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
- String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
- final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
- nativeNonSort.waitForCompletion(true);
-
- Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
- inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
- final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
- hadoopWithSort.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
- normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration configuration = new ScenarioConfiguration();
- configuration.addNonSortTestConf();
- final FileSystem fs = FileSystem.get(configuration);
- final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
- if (!fs.exists(path)) {
- new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
- Text.class.getName(), configuration).createSequenceTestFile(path.toString());
- }
- fs.close();
- }
-
- private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
- final FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputpath))) {
- fs.delete(new Path(outputpath), true);
- }
- fs.close();
- final Job job = new Job(conf, jobName);
- job.setJarByClass(NonSortTestMR.class);
- job.setMapperClass(NonSortTestMR.Map.class);
- job.setReducerClass(NonSortTestMR.KeyHashSumReduce.class);
- job.setOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setOutputValueClass(LongWritable.class);
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- return job;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
deleted file mode 100644
index 4ca2449..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class NonSortTestMR {
-
- public static class Map extends Mapper<Object, Text, Text, IntWritable> {
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
- final String line = value.toString();
- final StringTokenizer tokenizer = new StringTokenizer(line);
- while (tokenizer.hasMoreTokens()) {
- word.set(tokenizer.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class KeyHashSumReduce extends Reducer<Text, IntWritable, Text, LongWritable> {
- long sum = 0;
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(os);
-
- @Override
- public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
- InterruptedException {
- for (final IntWritable val : values) {
- os.reset();
- key.write(dos);
- final int hash = Arrays.hashCode(os.toByteArray());
- sum += hash;
- }
- }
-
- @Override
- public void cleanup(Context context) throws IOException, InterruptedException {
- context.write(new Text("NonSortTest"), new LongWritable(sum));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
deleted file mode 100644
index 5185371..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.util.Random;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-
-public class BytesFactory {
- public static Random r = new Random();
-
- public static Object newObject(byte[] seed, String className) {
- r.setSeed(seed.hashCode());
- if (className.equals(IntWritable.class.getName())) {
- return new IntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(FloatWritable.class.getName())) {
- return new FloatWritable(r.nextFloat());
- } else if (className.equals(DoubleWritable.class.getName())) {
- return new DoubleWritable(r.nextDouble());
- } else if (className.equals(LongWritable.class.getName())) {
- return new LongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(VIntWritable.class.getName())) {
- return new VIntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(VLongWritable.class.getName())) {
- return new VLongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(BooleanWritable.class.getName())) {
- return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
- } else if (className.equals(Text.class.getName())) {
- return new Text(BytesUtil.toStringBinary(seed));
- } else if (className.equals(ByteWritable.class.getName())) {
- return new ByteWritable(seed.length > 0 ? seed[0] : 0);
- } else if (className.equals(BytesWritable.class.getName())) {
- return new BytesWritable(seed);
- } else if (className.equals(UTF8.class.getName())) {
- return new UTF8(BytesUtil.toStringBinary(seed));
- } else if (className.equals(MockValueClass.class.getName())) {
- return new MockValueClass(seed);
- } else {
- return null;
- }
- }
-
-
- public static <VTYPE> byte[] fromBytes(byte[] bytes) throws Exception {
- throw new Exception("Not supported");
- }
-
- public static <VTYPE> byte[] toBytes(VTYPE obj) {
- final String className = obj.getClass().getName();
- if (className.equals(IntWritable.class.getName())) {
- return Ints.toByteArray(((IntWritable) obj).get());
- } else if (className.equals(FloatWritable.class.getName())) {
- return BytesUtil.toBytes(((FloatWritable) obj).get());
- } else if (className.equals(DoubleWritable.class.getName())) {
- return BytesUtil.toBytes(((DoubleWritable) obj).get());
- } else if (className.equals(LongWritable.class.getName())) {
- return Longs.toByteArray(((LongWritable) obj).get());
- } else if (className.equals(VIntWritable.class.getName())) {
- return Ints.toByteArray(((VIntWritable) obj).get());
- } else if (className.equals(VLongWritable.class.getName())) {
- return Longs.toByteArray(((VLongWritable) obj).get());
- } else if (className.equals(BooleanWritable.class.getName())) {
- return BytesUtil.toBytes(((BooleanWritable) obj).get());
- } else if (className.equals(Text.class.getName())) {
- return ((Text)obj).copyBytes();
- } else if (className.equals(ByteWritable.class.getName())) {
- return Ints.toByteArray((int) ((ByteWritable) obj).get());
- } else if (className.equals(BytesWritable.class.getName())) {
- // TODO: copyBytes instead?
- return ((BytesWritable) obj).getBytes();
- } else {
- return new byte[0];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
deleted file mode 100644
index a0f7d64..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
-
-public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
- private boolean nativetaskloaded = false;
-
- @Override
- public void init(Context context)
- throws IOException, ClassNotFoundException {
- try {
- super.init(context);
- nativetaskloaded = true;
- } catch (final Exception e) {
- nativetaskloaded = false;
- System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
- e.printStackTrace();
- }
- }
-
- @Override
- public void collect(K key, V value, int partition) throws IOException, InterruptedException {
- if (this.nativetaskloaded) {
- super.collect(key, value, partition);
- } else {
- // nothing to do.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
deleted file mode 100644
index b27f00f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-public class MockValueClass implements Writable {
- private final static int DEFAULT_ARRAY_LENGTH = 16;
- private int a = 0;
- private byte[] array;
- private final LongWritable longWritable;
- private final Text txt;
- private final Random rand = new Random();
-
- public MockValueClass() {
- a = rand.nextInt();
- array = new byte[DEFAULT_ARRAY_LENGTH];
- rand.nextBytes(array);
- longWritable = new LongWritable(rand.nextLong());
- txt = new Text(BytesUtil.toStringBinary(array));
- }
-
- public MockValueClass(byte[] seed) {
- a = seed.length;
- array = new byte[seed.length];
- System.arraycopy(seed, 0, array, 0, seed.length);
- longWritable = new LongWritable(a);
- txt = new Text(BytesUtil.toStringBinary(array));
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(a);
- out.writeInt(array.length);
- out.write(array);
- longWritable.write(out);
- txt.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- a = in.readInt();
- final int length = in.readInt();
- array = new byte[length];
- in.readFully(array);
- longWritable.readFields(in);
- txt.readFields(in);
- }
-}
[47/50] [abbrv] git commit: YARN-2542. Fixed NPE when retrieving
ApplicationReport from TimeLineServer. Contributed by Zhijie Shen
Posted by ar...@apache.org.
YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0ad975e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0ad975e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0ad975e
Branch: refs/heads/HDFS-6581
Commit: a0ad975ea1e70f9532cf6cb6c1d9d92736ca0ebc
Parents: e65ae57
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 12 15:24:17 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Sep 12 15:27:13 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 12 ++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 80 ++++++++++----------
3 files changed, 54 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index efc3e09..06d94ca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -348,6 +348,9 @@ Release 2.6.0 - UNRELEASED
YARN-2456. Possible livelock in CapacityScheduler when RM is recovering apps.
(Jian He via xgong)
+ YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
+ (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 54cfe91..a847cd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -463,9 +463,15 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.println(appReport.getHost());
appReportStr.print("\tAggregate Resource Allocation : ");
- ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport();
- appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
- appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
+ ApplicationResourceUsageReport usageReport =
+ appReport.getApplicationResourceUsageReport();
+ if (usageReport != null) {
+ //completed app report in the timeline server doesn't have usage report
+ appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
+ appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
+ } else {
+ appReportStr.println("N/A");
+ }
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 1a593d2..980517f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -86,44 +86,48 @@ public class TestYarnCLI {
@Test
public void testGetApplicationReport() throws Exception {
- ApplicationCLI cli = createAndGetAppCLI();
- ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- ApplicationResourceUsageReport usageReport =
- ApplicationResourceUsageReport.newInstance(
- 2, 0, null, null, null, 123456, 4567);
- ApplicationReport newApplicationReport = ApplicationReport.newInstance(
- applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
- "user", "queue", "appname", "host", 124, null,
- YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
- null);
- when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
- newApplicationReport);
- int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
- assertEquals(0, result);
- verify(client).getApplicationReport(applicationId);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintWriter pw = new PrintWriter(baos);
- pw.println("Application Report : ");
- pw.println("\tApplication-Id : application_1234_0005");
- pw.println("\tApplication-Name : appname");
- pw.println("\tApplication-Type : YARN");
- pw.println("\tUser : user");
- pw.println("\tQueue : queue");
- pw.println("\tStart-Time : 0");
- pw.println("\tFinish-Time : 0");
- pw.println("\tProgress : 53.79%");
- pw.println("\tState : FINISHED");
- pw.println("\tFinal-State : SUCCEEDED");
- pw.println("\tTracking-URL : N/A");
- pw.println("\tRPC Port : 124");
- pw.println("\tAM Host : host");
- pw.println("\tAggregate Resource Allocation : 123456 MB-seconds, 4567 vcore-seconds");
- pw.println("\tDiagnostics : diagnostics");
- pw.close();
- String appReportStr = baos.toString("UTF-8");
- Assert.assertEquals(appReportStr, sysOutStream.toString());
- verify(sysOut, times(1)).println(isA(String.class));
+ for (int i = 0; i < 2; ++i) {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationResourceUsageReport usageReport = i == 0 ? null :
+ ApplicationResourceUsageReport.newInstance(
+ 2, 0, null, null, null, 123456, 4567);
+ ApplicationReport newApplicationReport = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
+ null);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+ newApplicationReport);
+ int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
+ assertEquals(0, result);
+ verify(client, times(1 + i)).getApplicationReport(applicationId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Application Report : ");
+ pw.println("\tApplication-Id : application_1234_0005");
+ pw.println("\tApplication-Name : appname");
+ pw.println("\tApplication-Type : YARN");
+ pw.println("\tUser : user");
+ pw.println("\tQueue : queue");
+ pw.println("\tStart-Time : 0");
+ pw.println("\tFinish-Time : 0");
+ pw.println("\tProgress : 53.79%");
+ pw.println("\tState : FINISHED");
+ pw.println("\tFinal-State : SUCCEEDED");
+ pw.println("\tTracking-URL : N/A");
+ pw.println("\tRPC Port : 124");
+ pw.println("\tAM Host : host");
+ pw.println("\tAggregate Resource Allocation : " +
+ (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tDiagnostics : diagnostics");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ sysOutStream.reset();
+ verify(sysOut, times(1 + i)).println(isA(String.class));
+ }
}
@Test
[39/50] [abbrv] git commit: Merge remote-tracking branch
'apache/trunk' into MR-2841
Posted by ar...@apache.org.
Merge remote-tracking branch 'apache/trunk' into MR-2841
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b3f1e2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b3f1e2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b3f1e2c
Branch: refs/heads/HDFS-6581
Commit: 4b3f1e2ce48124b7406d77cc2ae1d0914311b0d4
Parents: 683987b c6107f5
Author: Todd Lipcon <to...@apache.org>
Authored: Fri Sep 5 10:47:27 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 5 10:47:27 2014 -0700
----------------------------------------------------------------------
LICENSE.txt | 290 ++++++++++++++++
NOTICE.txt | 2 +
README.txt | 31 ++
dev-support/create-release.sh | 24 +-
.../main/resources/assemblies/hadoop-src.xml | 8 +
hadoop-common-project/hadoop-common/CHANGES.txt | 20 ++
hadoop-common-project/hadoop-common/LICENSE.txt | 290 ----------------
hadoop-common-project/hadoop-common/NOTICE.txt | 2 -
hadoop-common-project/hadoop-common/README.txt | 31 --
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../hadoop/crypto/key/KeyProviderFactory.java | 36 +-
.../crypto/key/kms/KMSClientProvider.java | 57 +---
.../hadoop/crypto/random/OsSecureRandom.java | 21 +-
.../security/authorize/AccessControlList.java | 12 +-
.../DelegationTokenAuthenticationFilter.java | 15 +-
.../DelegationTokenAuthenticationHandler.java | 6 +-
.../web/DelegationTokenAuthenticator.java | 20 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 185 ++++++++++
.../apache/hadoop/crypto/TestCryptoCodec.java | 69 +++-
.../apache/hadoop/crypto/TestCryptoStreams.java | 2 +-
.../crypto/key/TestKeyProviderFactory.java | 13 +
.../crypto/random/TestOsSecureRandom.java | 15 +
...tionTokenAuthenticationHandlerWithMocks.java | 35 +-
.../hadoop/util/TestHttpExceptionUtils.java | 167 +++++++++
hadoop-common-project/hadoop-kms/pom.xml | 1 +
.../hadoop/crypto/key/kms/server/KMS.java | 27 +-
.../hadoop/crypto/key/kms/server/KMSACLs.java | 55 ++-
.../key/kms/server/KMSExceptionsProvider.java | 12 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 88 ++++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 100 +++++-
.../crypto/key/kms/server/TestKMSACLs.java | 2 +-
hadoop-dist/pom.xml | 3 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 70 ++--
.../hadoop/fs/http/client/HttpFSUtils.java | 50 ---
.../hadoop/lib/wsrs/ExceptionProvider.java | 14 +-
.../fs/http/client/BaseTestHttpFSWith.java | 4 +-
.../fs/http/server/TestHttpFSServerNoACLs.java | 10 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 25 ++
hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt | 271 ---------------
hadoop-hdfs-project/hadoop-hdfs/NOTICE.txt | 2 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 80 ++++-
.../hadoop/hdfs/DistributedFileSystem.java | 11 +-
.../org/apache/hadoop/hdfs/inotify/Event.java | 12 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +-
.../hdfs/server/datanode/BlockPoolManager.java | 12 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 44 ++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hdfs/server/namenode/FSEditLogOp.java | 18 +
.../server/namenode/FSImageSerialization.java | 17 +
.../hdfs/server/namenode/FSNamesystem.java | 60 ++--
.../hdfs/server/namenode/INodeReference.java | 4 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 1 +
.../server/namenode/NameNodeLayoutVersion.java | 4 +-
.../snapshot/DirectorySnapshottableFeature.java | 2 +-
.../org/apache/hadoop/hdfs/tools/GetConf.java | 4 +-
.../hadoop-hdfs/src/main/proto/inotify.proto | 1 +
.../src/main/resources/hdfs-default.xml | 18 +
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 18 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 8 +-
.../org/apache/hadoop/hdfs/TestDFSRename.java | 6 +
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 26 ++
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 119 +++++++
.../org/apache/hadoop/hdfs/TestLocalDFS.java | 29 +-
.../server/datanode/TestBlockPoolManager.java | 22 ++
.../hdfs/server/namenode/CreateEditsLog.java | 2 +-
.../hdfs/server/namenode/TestEditLog.java | 2 +-
.../hdfs/server/namenode/TestStartup.java | 2 +-
.../snapshot/TestSnapshotDiffReport.java | 36 ++
.../apache/hadoop/hdfs/tools/TestGetConf.java | 26 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4992 -> 5252 bytes
.../src/test/resources/editsStored.xml | 182 ++++++----
hadoop-mapreduce-project/CHANGES.txt | 6 +
hadoop-mapreduce-project/LICENSE.txt | 341 -------------------
hadoop-mapreduce-project/NOTICE.txt | 2 -
.../mapreduce/v2/app/job/impl/JobImpl.java | 2 +
.../java/org/apache/hadoop/mapred/MapTask.java | 4 +-
hadoop-yarn-project/CHANGES.txt | 12 +
hadoop-yarn-project/LICENSE.txt | 341 -------------------
hadoop-yarn-project/NOTICE.txt | 17 -
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../ApplicationHistoryServer.java | 19 +-
.../timeline/webapp/CrossOriginFilter.java | 10 +-
.../timeline/webapp/TestCrossOriginFilter.java | 15 +
.../nodemanager/LinuxContainerExecutor.java | 11 +
.../nodemanager/TestLinuxContainerExecutor.java | 60 ++++
.../scheduler/fair/AllocationConfiguration.java | 22 +-
.../fair/AllocationFileLoaderService.java | 48 ++-
.../scheduler/fair/FSLeafQueue.java | 54 ++-
.../scheduler/fair/FSParentQueue.java | 6 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 23 +-
.../scheduler/fair/FairScheduler.java | 60 +---
.../scheduler/fair/QueueManager.java | 32 +-
.../fair/TestAllocationFileLoaderService.java | 45 ++-
.../scheduler/fair/TestFSLeafQueue.java | 198 +++++++++--
.../scheduler/fair/TestFairScheduler.java | 158 ++-------
.../src/site/apt/FairScheduler.apt.vm | 10 +
103 files changed, 2395 insertions(+), 1999 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] git commit: MAPREDUCE-6054. native-task: Speed up
tests. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-6054. native-task: Speed up tests. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfd1d75d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfd1d75d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfd1d75d
Branch: refs/heads/HDFS-6581
Commit: bfd1d75d875b6ba261fdb1825d0f151b026c2d24
Parents: fad4524
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Aug 27 12:25:07 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Aug 27 12:25:49 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 2 +-
.../mapred/nativetask/util/BytesUtil.java | 7 +-
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 8 +-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 94 +++++++++-----------
.../mapred/nativetask/kvtest/TestInputFile.java | 13 ++-
.../nativetask/testutil/BytesFactory.java | 71 +++++++++------
.../nativetask/testutil/MockValueClass.java | 8 ++
.../mapred/nativetask/utils/TestBytesUtil.java | 7 ++
8 files changed, 123 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 4dc08cb..6384757 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -16,4 +16,4 @@ MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (B
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd)
MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
-
+MAPREDUCE-6054. native-task: Speed up tests (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index d90ae8d..e33b23e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -23,6 +23,9 @@ import com.google.common.primitives.Longs;
public class BytesUtil {
+ private static final char[] HEX_CHARS =
+ "0123456789abcdef".toCharArray();
+
/**
* Converts a big-endian byte array to a long value.
*
@@ -124,7 +127,9 @@ public class BytesUtil {
|| " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
result.append((char)ch);
} else {
- result.append(String.format("\\x%02X", ch));
+ result.append("\\x");
+ result.append(HEX_CHARS[(ch >> 4) & 0x0F]);
+ result.append(HEX_CHARS[ch & 0x0F]);
}
}
return result.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 6d683f8..3215d0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.mapred.nativetask.kvtest;
import java.io.IOException;
import java.util.zip.CRC32;
+import com.google.common.base.Stopwatch;
import com.google.common.primitives.Longs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -36,6 +40,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class KVJob {
public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
+ private static final Log LOG = LogFactory.getLog(KVJob.class);
Job job = null;
public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
@@ -82,8 +87,9 @@ public class KVJob {
final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
TestConstants.FILESIZE_KEY, "1000")),
keyclass.getName(), valueclass.getName(), conf);
+ Stopwatch sw = new Stopwatch().start();
testfile.createSequenceTestFile(inputpath);
-
+ LOG.info("Created test file " + inputpath + " in " + sw.elapsedMillis() + "ms");
}
job.setInputFormatClass(SequenceFileInputFormat.class);
FileInputFormat.addInputPath(job, new Path(inputpath));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 7c4f3af..893a422 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -20,32 +20,29 @@ package org.apache.hadoop.mapred.nativetask.kvtest;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
@RunWith(Parameterized.class)
public class KVTest {
private static final Log LOG = LogFactory.getLog(KVTest.class);
- private static Class<?>[] keyclasses = null;
- private static Class<?>[] valueclasses = null;
- private static String[] keyclassNames = null;
- private static String[] valueclassNames = null;
-
private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();
static {
@@ -53,50 +50,46 @@ public class KVTest {
hadoopkvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
}
- @Parameters(name = "key:{0}\nvalue:{1}")
- public static Iterable<Class<?>[]> data() {
- final String valueclassesStr = nativekvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
- LOG.info(valueclassesStr);
- valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < valueclassNames.length; i++) {
+ private static List<Class<?>> parseClassNames(String spec) {
+ List<Class<?>> ret = Lists.newArrayList();
+ Iterable<String> classNames = Splitter.on(';').trimResults()
+ .omitEmptyStrings().split(spec);
+ for (String className : classNames) {
try {
- if (valueclassNames[i].equals("")) {
- continue;
- }
- tmpvalueclasses.add(Class.forName(valueclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
+ ret.add(Class.forName(className));
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
}
}
- valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
- final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
- LOG.info(keyclassesStr);
- keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < keyclassNames.length; i++) {
- try {
- if (keyclassNames[i].equals("")) {
- continue;
- }
- tmpkeyclasses.add(Class.forName(keyclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
- }
+ return ret;
+ }
+
+ /**
+ * Parameterize the test with the specified key and value types.
+ */
+ @Parameters(name = "key:{0}\nvalue:{1}")
+ public static Iterable<Class<?>[]> data() throws Exception {
+ // Parse the config.
+ final String valueClassesStr = nativekvtestconf
+ .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
+ LOG.info("Parameterizing with value classes: " + valueClassesStr);
+ List<Class<?>> valueClasses = parseClassNames(valueClassesStr);
+
+ final String keyClassesStr = nativekvtestconf.get(
+ TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
+ LOG.info("Parameterizing with key classes: " + keyClassesStr);
+ List<Class<?>> keyClasses = parseClassNames(keyClassesStr);
+
+ // Generate an entry for each key type.
+ List<Class<?>[]> pairs = Lists.newArrayList();
+ for (Class<?> keyClass : keyClasses) {
+ pairs.add(new Class<?>[]{ keyClass, LongWritable.class });
}
- keyclasses = tmpkeyclasses.toArray(new Class[tmpkeyclasses.size()]);
- final Class<?>[][] kvgroup = new Class<?>[keyclassNames.length * valueclassNames.length][2];
- for (int i = 0; i < keyclassNames.length; i++) {
- final int tmpindex = i * valueclassNames.length;
- for (int j = 0; j < valueclassNames.length; j++) {
- kvgroup[tmpindex + j][0] = keyclasses[i];
- kvgroup[tmpindex + j][1] = valueclasses[j];
- }
+ // ...and for each value type.
+ for (Class<?> valueClass : valueClasses) {
+ pairs.add(new Class<?>[]{ LongWritable.class, valueClass });
}
- return Arrays.asList(kvgroup);
+ return pairs;
}
private final Class<?> keyclass;
@@ -105,12 +98,10 @@ public class KVTest {
public KVTest(Class<?> keyclass, Class<?> valueclass) {
this.keyclass = keyclass;
this.valueclass = valueclass;
-
}
@Test
public void testKVCompability() {
-
try {
final String nativeoutput = this.runNativeTest(
"Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
@@ -135,11 +126,6 @@ public class KVTest {
}
}
- @Before
- public void startUp() {
-
- }
-
private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+ keyclass.getName()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index 80e1055..4c77f4c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.VIntWritable;
import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
@@ -123,7 +124,7 @@ public class TestInputFile {
}
public void createSequenceTestFile(String filepath, int base, byte start) throws Exception {
- LOG.info("create file " + filepath);
+ LOG.info("creating file " + filepath + "(" + filesize + " bytes)");
LOG.info(keyClsName + " " + valueClsName);
Class<?> tmpkeycls, tmpvaluecls;
try {
@@ -178,6 +179,9 @@ public class TestInputFile {
int valuebytesnum = 0;
int offset = 0;
+ Writable keyWritable = BytesFactory.newObject(null, keyClsName);
+ Writable valWritable = BytesFactory.newObject(null, valueClsName);
+
while (offset < buflen) {
final int remains = buflen - offset;
keybytesnum = keyMaxBytesNum;
@@ -202,9 +206,12 @@ public class TestInputFile {
System.arraycopy(databuf, offset, value, 0, valuebytesnum);
offset += valuebytesnum;
-
+
+ BytesFactory.updateObject(keyWritable, key);
+ BytesFactory.updateObject(valWritable, value);
+
try {
- writer.append(BytesFactory.newObject(key, this.keyClsName), BytesFactory.newObject(value, this.valueClsName));
+ writer.append(keyWritable, valWritable);
} catch (final IOException e) {
e.printStackTrace();
throw new Exception("sequence file create failed", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
index 5185371..bee2f0c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.mapred.nativetask.testutil;
import java.util.Random;
+import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
+
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -32,43 +34,58 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.UTF8;
import org.apache.hadoop.io.VIntWritable;
import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
public class BytesFactory {
public static Random r = new Random();
- public static Object newObject(byte[] seed, String className) {
- r.setSeed(seed.hashCode());
- if (className.equals(IntWritable.class.getName())) {
- return new IntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(FloatWritable.class.getName())) {
- return new FloatWritable(r.nextFloat());
- } else if (className.equals(DoubleWritable.class.getName())) {
- return new DoubleWritable(r.nextDouble());
- } else if (className.equals(LongWritable.class.getName())) {
- return new LongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(VIntWritable.class.getName())) {
- return new VIntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(VLongWritable.class.getName())) {
- return new VLongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(BooleanWritable.class.getName())) {
- return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
- } else if (className.equals(Text.class.getName())) {
- return new Text(BytesUtil.toStringBinary(seed));
- } else if (className.equals(ByteWritable.class.getName())) {
- return new ByteWritable(seed.length > 0 ? seed[0] : 0);
- } else if (className.equals(BytesWritable.class.getName())) {
- return new BytesWritable(seed);
- } else if (className.equals(UTF8.class.getName())) {
- return new UTF8(BytesUtil.toStringBinary(seed));
- } else if (className.equals(MockValueClass.class.getName())) {
- return new MockValueClass(seed);
+ public static void updateObject(Writable obj, byte[] seed) {
+ if (obj instanceof IntWritable) {
+ ((IntWritable)obj).set(Ints.fromByteArray(seed));
+ } else if (obj instanceof FloatWritable) {
+ ((FloatWritable)obj).set(r.nextFloat());
+ } else if (obj instanceof DoubleWritable) {
+ ((DoubleWritable)obj).set(r.nextDouble());
+ } else if (obj instanceof LongWritable) {
+ ((LongWritable)obj).set(Longs.fromByteArray(seed));
+ } else if (obj instanceof VIntWritable) {
+ ((VIntWritable)obj).set(Ints.fromByteArray(seed));
+ } else if (obj instanceof VLongWritable) {
+ ((VLongWritable)obj).set(Longs.fromByteArray(seed));
+ } else if (obj instanceof BooleanWritable) {
+ ((BooleanWritable)obj).set(seed[0] % 2 == 1 ? true : false);
+ } else if (obj instanceof Text) {
+ ((Text)obj).set(BytesUtil.toStringBinary(seed));
+ } else if (obj instanceof ByteWritable) {
+ ((ByteWritable)obj).set(seed.length > 0 ? seed[0] : 0);
+ } else if (obj instanceof BytesWritable) {
+ ((BytesWritable)obj).set(seed, 0, seed.length);
+ } else if (obj instanceof UTF8) {
+ ((UTF8)obj).set(BytesUtil.toStringBinary(seed));
+ } else if (obj instanceof MockValueClass) {
+ ((MockValueClass)obj).set(seed);
} else {
- return null;
+ throw new IllegalArgumentException("unknown writable: " +
+ obj.getClass().getName());
}
}
+ public static Writable newObject(byte[] seed, String className) {
+ Writable ret;
+ try {
+ Class<?> clazz = Class.forName(className);
+ Preconditions.checkArgument(Writable.class.isAssignableFrom(clazz));
+ ret = (Writable)clazz.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ if (seed != null) {
+ updateObject(ret, seed);
+ }
+ return ret;
+ }
public static <VTYPE> byte[] fromBytes(byte[] bytes) throws Exception {
throw new Exception("Not supported");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
index b27f00f..a476a2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
@@ -51,6 +51,14 @@ public class MockValueClass implements Writable {
txt = new Text(BytesUtil.toStringBinary(array));
}
+ public void set(byte[] seed) {
+ a = seed.length;
+ array = new byte[seed.length];
+ System.arraycopy(seed, 0, array, 0, seed.length);
+ longWritable.set(a);
+ txt.set(BytesUtil.toStringBinary(array));
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(a);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
index 57b77cd..c09f08c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/utils/TestBytesUtil.java
@@ -53,4 +53,11 @@ public class TestBytesUtil extends TestCase {
Assert.assertEquals(d, BytesUtil.toDouble(doubleBytes));
}
+
+ public void testToStringBinary() {
+ Assert.assertEquals("\\x01\\x02ABC",
+ BytesUtil.toStringBinary(new byte[] { 1, 2, 65, 66, 67 }));
+ Assert.assertEquals("\\x10\\x11",
+ BytesUtil.toStringBinary(new byte[] { 16, 17 }));
+ }
}
[43/50] [abbrv] git commit: YARN-2547. Cross Origin Filter throws
UnsupportedOperationException upon destroy (Mit Desai via jeagles)
Posted by ar...@apache.org.
YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon destroy (Mit Desai via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54e57948
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54e57948
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54e57948
Branch: refs/heads/HDFS-6581
Commit: 54e5794806bd856da0277510efe63656eed23146
Parents: 3122daa
Author: Jonathan Eagles <je...@gmail.com>
Authored: Fri Sep 12 15:35:17 2014 -0500
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Fri Sep 12 15:35:17 2014 -0500
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../timeline/webapp/CrossOriginFilter.java | 12 ++---
.../timeline/webapp/TestCrossOriginFilter.java | 49 ++++++++++++++++++++
3 files changed, 58 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bb73dc7..3993f5e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -214,6 +214,9 @@ Release 2.6.0 - UNRELEASED
YARN-2229. Changed the integer field of ContainerId to be long type.
(Tsuyoshi OZAWA via jianhe)
+ YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon
+ destroy (Mit Desai via jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index 5a0703d..cceee54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -150,8 +150,8 @@ public class CrossOriginFilter implements Filter {
if (allowedMethodsConfig == null) {
allowedMethodsConfig = ALLOWED_METHODS_DEFAULT;
}
- allowedMethods =
- Arrays.asList(allowedMethodsConfig.trim().split("\\s*,\\s*"));
+ allowedMethods.addAll(
+ Arrays.asList(allowedMethodsConfig.trim().split("\\s*,\\s*")));
LOG.info("Allowed Methods: " + getAllowedMethodsHeader());
}
@@ -161,8 +161,8 @@ public class CrossOriginFilter implements Filter {
if (allowedHeadersConfig == null) {
allowedHeadersConfig = ALLOWED_HEADERS_DEFAULT;
}
- allowedHeaders =
- Arrays.asList(allowedHeadersConfig.trim().split("\\s*,\\s*"));
+ allowedHeaders.addAll(
+ Arrays.asList(allowedHeadersConfig.trim().split("\\s*,\\s*")));
LOG.info("Allowed Headers: " + getAllowedHeadersHeader());
}
@@ -172,8 +172,8 @@ public class CrossOriginFilter implements Filter {
if (allowedOriginsConfig == null) {
allowedOriginsConfig = ALLOWED_ORIGINS_DEFAULT;
}
- allowedOrigins =
- Arrays.asList(allowedOriginsConfig.trim().split("\\s*,\\s*"));
+ allowedOrigins.addAll(
+ Arrays.asList(allowedOriginsConfig.trim().split("\\s*,\\s*")));
allowAllOrigins = allowedOrigins.contains("*");
LOG.info("Allowed Origins: " + StringUtils.join(allowedOrigins, ','));
LOG.info("Allow All Origins: " + allowAllOrigins);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index ccc9bbf..e0990f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
@@ -217,6 +217,55 @@ public class TestCrossOriginFilter {
verify(mockChain).doFilter(mockReq, mockRes);
}
+ @Test
+ public void testCrossOriginFilterAfterRestart() throws ServletException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "example.com");
+ conf.put(CrossOriginFilter.ALLOWED_HEADERS, "X-Requested-With,Accept");
+ conf.put(CrossOriginFilter.ALLOWED_METHODS, "GET,POST");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ //verify filter values
+ Assert.assertTrue("Allowed headers do not match",
+ filter.getAllowedHeadersHeader()
+ .compareTo("X-Requested-With,Accept") == 0);
+ Assert.assertTrue("Allowed methods do not match",
+ filter.getAllowedMethodsHeader()
+ .compareTo("GET,POST") == 0);
+ Assert.assertTrue(filter.isOriginAllowed("example.com"));
+
+ //destroy filter values and clear conf
+ filter.destroy();
+ conf.clear();
+
+ // Setup the configuration settings of the server
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "newexample.com");
+ conf.put(CrossOriginFilter.ALLOWED_HEADERS, "Content-Type,Origin");
+ conf.put(CrossOriginFilter.ALLOWED_METHODS, "GET,HEAD");
+ filterConfig = new FilterConfigTest(conf);
+
+ //initialize filter
+ filter.init(filterConfig);
+
+ //verify filter values
+ Assert.assertTrue("Allowed headers do not match",
+ filter.getAllowedHeadersHeader()
+ .compareTo("Content-Type,Origin") == 0);
+ Assert.assertTrue("Allowed methods do not match",
+ filter.getAllowedMethodsHeader()
+ .compareTo("GET,HEAD") == 0);
+ Assert.assertTrue(filter.isOriginAllowed("newexample.com"));
+
+ //destroy filter values
+ filter.destroy();
+ }
+
private static class FilterConfigTest implements FilterConfig {
final Map<String, String> map;
[32/50] [abbrv] MAPREDUCE-6055. native-task: findbugs,
interface annotations, and other misc cleanup
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
index ec326ca..d8a6595 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class LongWritableSerializer extends DefaultSerializer implements
INativeComparable {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
index f5a033d..5881a46 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
@@ -21,8 +21,10 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
+@InterfaceAudience.Private
public class NativeSerialization {
private final ConcurrentHashMap<String, Class<?>> map = new ConcurrentHashMap<String, Class<?>>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
index afa4e8e..f6e7cf5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class NullWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
index e95a0c4..8dee58f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.mapred.nativetask.serde;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
public enum SerializationFramework {
WRITABLE_SERIALIZATION(0), NATIVE_SERIALIZATION(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
index cde4c7b..e702dfe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
@@ -22,9 +22,11 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class TextSerializer implements INativeSerializer<Text>, INativeComparable {
public TextSerializer() throws SecurityException, NoSuchMethodException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VIntWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VIntWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VIntWritableSerializer.java
index 0e142f3..0cbb9ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VIntWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VIntWritableSerializer.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.mapred.nativetask.serde;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class VIntWritableSerializer extends DefaultSerializer implements
INativeComparable {
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VLongWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VLongWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VLongWritableSerializer.java
index d66e179..af40b77 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VLongWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/VLongWritableSerializer.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.mapred.nativetask.serde;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class VLongWritableSerializer extends DefaultSerializer implements
INativeComparable {
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index e33b23e..35ef0a9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.mapred.nativetask.util;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
public class BytesUtil {
private static final char[] HEX_CHARS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
index d7cf00a..7870ac7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ConfigUtil.java
@@ -24,7 +24,10 @@ import java.util.Map;
import com.google.common.base.Charsets;
import org.apache.hadoop.conf.Configuration;
-public class ConfigUtil {
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public abstract class ConfigUtil {
public static byte[][] toBytes(Configuration conf) {
List<byte[]> nativeConfigs = new ArrayList<byte[]>();
for (Map.Entry<String, String> e : conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
index 77eae50..6d63be5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
@@ -19,12 +19,14 @@ package org.apache.hadoop.mapred.nativetask.util;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskID;
+@InterfaceAudience.Private
public class LocalJobOutputFiles implements NativeTaskOutput {
static final String TASKTRACKER_OUTPUT = "output";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
index 3da8a0d..e92901b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred.nativetask.util;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskID;
@@ -26,6 +27,7 @@ import org.apache.hadoop.mapred.TaskID;
/**
* base class of output files manager.
*/
+@InterfaceAudience.Private
public interface NativeTaskOutput {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
index 18cbddb..5cad10b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred.nativetask.util;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
@@ -33,7 +34,7 @@ import org.apache.hadoop.mapred.TaskID;
* mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used
* from TaskTracker space.
*/
-
+@InterfaceAudience.Private
public class NativeTaskOutputFiles implements NativeTaskOutput {
static final String TASKTRACKER_OUTPUT = "output";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
index bdef796..dd45ede 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
@@ -22,9 +22,10 @@ import java.lang.reflect.Constructor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-
+@InterfaceAudience.Private
public class OutputUtil {
private static Log LOG = LogFactory.getLog(OutputUtil.class);
@@ -40,7 +41,7 @@ public class OutputUtil {
NativeTaskOutput instance = (NativeTaskOutput) ctor.newInstance(conf, id);
return instance;
} catch (Exception e) {
- return null;
+ throw new RuntimeException(e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
index dd73c1e..017d834 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
@@ -15,16 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.mapred.nativetask.util;
import com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+@InterfaceAudience.Private
public class ReadWriteBuffer {
private byte[] _buff;
private int _writePoint;
private int _readPoint;
- final int CACHE_LINE_SIZE = 16;
+ final static int CACHE_LINE_SIZE = 16;
public ReadWriteBuffer(int length) {
if (length > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SizedWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SizedWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SizedWritable.java
index 5dc099d..284661f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SizedWritable.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SizedWritable.java
@@ -19,12 +19,14 @@ package org.apache.hadoop.mapred.nativetask.util;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
+@InterfaceAudience.Private
public class SizedWritable<T> {
- public static int INVALID_LENGTH = -1;
+ public static final int INVALID_LENGTH = -1;
public int length = INVALID_LENGTH;
public Writable v;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/TestTaskContext.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/TestTaskContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/TestTaskContext.java
index a76b1b2..508ded3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/TestTaskContext.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/TestTaskContext.java
@@ -30,13 +30,13 @@ public class TestTaskContext extends TestCase {
TaskContext context = new TaskContext(null, null, null, null, null, null, null);
context.setInputKeyClass(IntWritable.class);
- assertEquals(IntWritable.class.getName(), context.getInputKeyClass().getName());
+ assertEquals(IntWritable.class.getName(), context.getInputKeyClass().getName());
context.setInputValueClass(Text.class);
assertEquals(Text.class.getName(), context.getInputValueClass().getName());
context.setOutputKeyClass(LongWritable.class);
- assertEquals(LongWritable.class.getName(), context.getOuputKeyClass().getName());
+ assertEquals(LongWritable.class.getName(), context.getOutputKeyClass().getName());
context.setOutputValueClass(FloatWritable.class);
assertEquals(FloatWritable.class.getName(), context.getOutputValueClass().getName());
[50/50] [abbrv] git commit: Merge branch 'trunk' into HDFS-6581
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-6581
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebdd5e50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebdd5e50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebdd5e50
Branch: refs/heads/HDFS-6581
Commit: ebdd5e50d5819b4ec704c6d6156ba2944bbab513
Parents: 7aeda47 3c91817
Author: arp <ar...@apache.org>
Authored: Fri Sep 12 20:53:18 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Fri Sep 12 20:53:18 2014 -0700
----------------------------------------------------------------------
LICENSE.txt | 33 +
.../assemblies/hadoop-mapreduce-dist.xml | 7 +
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/crypto/key/TestKeyShell.java | 4 +-
.../hadoop/security/alias/TestCredShell.java | 22 +-
.../crypto/key/kms/server/KMSConfiguration.java | 15 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 3 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 3 +-
hadoop-mapreduce-project/CHANGES.txt | 65 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 216 +
.../src/CMakeLists.txt | 276 +
.../src/JNIFlags.cmake | 118 +
.../src/config.h.cmake | 23 +
.../hadoop/mapred/nativetask/Command.java | 57 +
.../mapred/nativetask/CommandDispatcher.java | 33 +
.../hadoop/mapred/nativetask/Constants.java | 48 +
.../hadoop/mapred/nativetask/DataChannel.java | 41 +
.../hadoop/mapred/nativetask/DataReceiver.java | 37 +
.../mapred/nativetask/HadoopPlatform.java | 84 +
.../mapred/nativetask/ICombineHandler.java | 44 +
.../mapred/nativetask/INativeComparable.java | 54 +
.../mapred/nativetask/INativeHandler.java | 53 +
.../mapred/nativetask/NativeBatchProcessor.java | 279 +
.../mapred/nativetask/NativeDataSource.java | 47 +
.../mapred/nativetask/NativeDataTarget.java | 47 +
.../NativeMapOutputCollectorDelegator.java | 171 +
.../hadoop/mapred/nativetask/NativeRuntime.java | 197 +
.../hadoop/mapred/nativetask/Platform.java | 100 +
.../hadoop/mapred/nativetask/Platforms.java | 79 +
.../mapred/nativetask/StatusReportChecker.java | 99 +
.../hadoop/mapred/nativetask/TaskContext.java | 94 +
.../mapred/nativetask/buffer/BufferType.java | 27 +
.../nativetask/buffer/ByteBufferDataReader.java | 148 +
.../nativetask/buffer/ByteBufferDataWriter.java | 169 +
.../nativetask/buffer/DataInputStream.java | 27 +
.../nativetask/buffer/DataOutputStream.java | 39 +
.../mapred/nativetask/buffer/InputBuffer.java | 136 +
.../mapred/nativetask/buffer/OutputBuffer.java | 76 +
.../nativetask/handlers/BufferPullee.java | 121 +
.../nativetask/handlers/BufferPuller.java | 203 +
.../nativetask/handlers/BufferPushee.java | 151 +
.../nativetask/handlers/BufferPusher.java | 91 +
.../nativetask/handlers/CombinerHandler.java | 145 +
.../mapred/nativetask/handlers/IDataLoader.java | 37 +
.../handlers/NativeCollectorOnlyHandler.java | 171 +
.../serde/BoolWritableSerializer.java | 35 +
.../serde/ByteWritableSerializer.java | 35 +
.../serde/BytesWritableSerializer.java | 48 +
.../nativetask/serde/DefaultSerializer.java | 71 +
.../serde/DoubleWritableSerializer.java | 35 +
.../serde/FloatWritableSerializer.java | 36 +
.../mapred/nativetask/serde/IKVSerializer.java | 48 +
.../nativetask/serde/INativeSerializer.java | 49 +
.../nativetask/serde/IntWritableSerializer.java | 35 +
.../mapred/nativetask/serde/KVSerializer.java | 115 +
.../serde/LongWritableSerializer.java | 34 +
.../nativetask/serde/NativeSerialization.java | 91 +
.../serde/NullWritableSerializer.java | 35 +
.../serde/SerializationFramework.java | 35 +
.../mapred/nativetask/serde/TextSerializer.java | 49 +
.../serde/VIntWritableSerializer.java | 27 +
.../serde/VLongWritableSerializer.java | 27 +
.../mapred/nativetask/util/BytesUtil.java | 172 +
.../mapred/nativetask/util/ConfigUtil.java | 47 +
.../nativetask/util/LocalJobOutputFiles.java | 159 +
.../nativetask/util/NativeTaskOutput.java | 106 +
.../nativetask/util/NativeTaskOutputFiles.java | 170 +
.../mapred/nativetask/util/OutputUtil.java | 47 +
.../mapred/nativetask/util/ReadWriteBuffer.java | 159 +
.../mapred/nativetask/util/SizedWritable.java | 55 +
.../src/main/native/gtest/gtest-all.cc | 9120 ++++++++
.../src/main/native/gtest/gtest_main.cc | 39 +
.../src/main/native/gtest/include/gtest/gtest.h | 19541 +++++++++++++++++
.../src/main/native/src/NativeTask.h | 507 +
.../src/main/native/src/codec/BlockCodec.cc | 179 +
.../src/main/native/src/codec/BlockCodec.h | 98 +
.../src/main/native/src/codec/GzipCodec.cc | 192 +
.../src/main/native/src/codec/GzipCodec.h | 82 +
.../src/main/native/src/codec/Lz4Codec.cc | 88 +
.../src/main/native/src/codec/Lz4Codec.h | 45 +
.../src/main/native/src/codec/SnappyCodec.cc | 98 +
.../src/main/native/src/codec/SnappyCodec.h | 46 +
.../native/src/handler/AbstractMapHandler.cc | 34 +
.../native/src/handler/AbstractMapHandler.h | 102 +
.../src/main/native/src/handler/BatchHandler.cc | 355 +
.../src/main/native/src/handler/BatchHandler.h | 157 +
.../main/native/src/handler/CombineHandler.cc | 254 +
.../main/native/src/handler/CombineHandler.h | 90 +
.../src/handler/MCollectorOutputHandler.cc | 98 +
.../src/handler/MCollectorOutputHandler.h | 53 +
.../src/main/native/src/lib/BufferStream.cc | 43 +
.../src/main/native/src/lib/BufferStream.h | 156 +
.../src/main/native/src/lib/Buffers.cc | 235 +
.../src/main/native/src/lib/Buffers.h | 637 +
.../src/main/native/src/lib/Combiner.h | 70 +
.../src/main/native/src/lib/Compressions.cc | 145 +
.../src/main/native/src/lib/Compressions.h | 108 +
.../src/main/native/src/lib/Constants.h | 26 +
.../src/main/native/src/lib/FileSystem.cc | 263 +
.../src/main/native/src/lib/FileSystem.h | 123 +
.../src/main/native/src/lib/IFile.cc | 230 +
.../src/main/native/src/lib/IFile.h | 166 +
.../src/main/native/src/lib/Iterator.cc | 91 +
.../src/main/native/src/lib/Iterator.h | 47 +
.../src/main/native/src/lib/Log.cc | 30 +
.../src/main/native/src/lib/Log.h | 48 +
.../main/native/src/lib/MapOutputCollector.cc | 413 +
.../main/native/src/lib/MapOutputCollector.h | 167 +
.../src/main/native/src/lib/MapOutputSpec.cc | 64 +
.../src/main/native/src/lib/MapOutputSpec.h | 81 +
.../src/main/native/src/lib/MemoryBlock.cc | 67 +
.../src/main/native/src/lib/MemoryBlock.h | 172 +
.../src/main/native/src/lib/MemoryPool.h | 90 +
.../src/main/native/src/lib/Merge.cc | 156 +
.../src/main/native/src/lib/Merge.h | 254 +
.../src/main/native/src/lib/MinHeap.h | 64 +
.../src/main/native/src/lib/NativeLibrary.cc | 89 +
.../src/main/native/src/lib/NativeLibrary.h | 57 +
.../main/native/src/lib/NativeObjectFactory.cc | 444 +
.../main/native/src/lib/NativeObjectFactory.h | 106 +
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 256 +
.../src/main/native/src/lib/NativeTask.cc | 211 +
.../src/main/native/src/lib/PartitionBucket.cc | 75 +
.../src/main/native/src/lib/PartitionBucket.h | 142 +
.../native/src/lib/PartitionBucketIterator.cc | 114 +
.../native/src/lib/PartitionBucketIterator.h | 54 +
.../src/main/native/src/lib/Path.cc | 53 +
.../src/main/native/src/lib/Path.h | 38 +
.../src/main/native/src/lib/SpillInfo.cc | 73 +
.../src/main/native/src/lib/SpillInfo.h | 106 +
.../main/native/src/lib/SpillOutputService.h | 44 +
.../src/main/native/src/lib/Streams.cc | 122 +
.../src/main/native/src/lib/Streams.h | 221 +
.../src/main/native/src/lib/TaskCounters.cc | 40 +
.../src/main/native/src/lib/TaskCounters.h | 44 +
.../src/main/native/src/lib/commons.h | 52 +
.../src/main/native/src/lib/jniutils.cc | 111 +
.../src/main/native/src/lib/jniutils.h | 57 +
.../src/main/native/src/lib/primitives.h | 287 +
.../src/main/native/src/util/Checksum.cc | 749 +
.../src/main/native/src/util/Checksum.h | 78 +
.../main/native/src/util/DualPivotQuickSort.h | 137 +
.../src/main/native/src/util/Random.cc | 285 +
.../src/main/native/src/util/Random.h | 140 +
.../src/main/native/src/util/StringUtil.cc | 213 +
.../src/main/native/src/util/StringUtil.h | 66 +
.../src/main/native/src/util/SyncUtils.cc | 55 +
.../src/main/native/src/util/SyncUtils.h | 71 +
.../src/main/native/src/util/Timer.cc | 116 +
.../src/main/native/src/util/Timer.h | 56 +
.../src/main/native/src/util/WritableUtils.cc | 309 +
.../src/main/native/src/util/WritableUtils.h | 124 +
.../src/main/native/test.sh | 36 +
.../src/main/native/test/TestCommand.cc | 39 +
.../src/main/native/test/TestCompressions.cc | 286 +
.../src/main/native/test/TestConfig.cc | 67 +
.../src/main/native/test/TestCounter.cc | 48 +
.../src/main/native/test/TestFileSystem.cc | 50 +
.../src/main/native/test/TestIFile.cc | 199 +
.../src/main/native/test/TestMain.cc | 116 +
.../src/main/native/test/TestPrimitives.cc | 304 +
.../src/main/native/test/TestSort.cc | 306 +
.../src/main/native/test/lib/TestByteArray.cc | 43 +
.../src/main/native/test/lib/TestByteBuffer.cc | 47 +
.../lib/TestComparatorForDualPivotQuickSort.cc | 89 +
.../native/test/lib/TestComparatorForStdSort.cc | 89 +
.../native/test/lib/TestFixSizeContainer.cc | 55 +
.../src/main/native/test/lib/TestIterator.cc | 114 +
.../src/main/native/test/lib/TestKVBuffer.cc | 54 +
.../native/test/lib/TestMemBlockIterator.cc | 110 +
.../src/main/native/test/lib/TestMemoryBlock.cc | 109 +
.../src/main/native/test/lib/TestMemoryPool.cc | 48 +
.../main/native/test/lib/TestPartitionBucket.cc | 222 +
.../src/main/native/test/lib/TestReadBuffer.cc | 69 +
.../main/native/test/lib/TestReadWriteBuffer.cc | 70 +
.../src/main/native/test/test_commons.cc | 344 +
.../src/main/native/test/test_commons.h | 150 +
.../src/main/native/test/util/TestChecksum.cc | 43 +
.../src/main/native/test/util/TestProcess.cc | 33 +
.../src/main/native/test/util/TestStringUtil.cc | 64 +
.../main/native/test/util/TestWritableUtils.cc | 49 +
.../main/native/testData/testGlibCBugSpill.out | 2 +
...org.apache.hadoop.mapred.nativetask.Platform | 14 +
.../mapred/nativetask/TestTaskContext.java | 44 +
.../nativetask/buffer/TestBufferPushPull.java | 279 +
.../buffer/TestByteBufferReadWrite.java | 168 +
.../nativetask/buffer/TestInputBuffer.java | 53 +
.../nativetask/buffer/TestOutputBuffer.java | 45 +
.../nativetask/combinertest/CombinerTest.java | 124 +
.../combinertest/LargeKVCombinerTest.java | 115 +
.../combinertest/OldAPICombinerTest.java | 129 +
.../nativetask/combinertest/WordCount.java | 91 +
.../combinertest/WordCountWithOldAPI.java | 68 +
.../nativetask/compresstest/CompressMapper.java | 63 +
.../nativetask/compresstest/CompressTest.java | 144 +
.../nativetask/handlers/TestCombineHandler.java | 80 +
.../TestNativeCollectorOnlyHandler.java | 129 +
.../nativetask/kvtest/HashSumReducer.java | 49 +
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 108 +
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 153 +
.../mapred/nativetask/kvtest/LargeKVTest.java | 131 +
.../mapred/nativetask/kvtest/TestInputFile.java | 227 +
.../nativetask/nonsorttest/NonSortTest.java | 118 +
.../nativetask/nonsorttest/NonSortTestMR.java | 72 +
.../nativetask/serde/TestKVSerializer.java | 148 +
.../serde/TestNativeSerialization.java | 114 +
.../nativetask/testutil/BytesFactory.java | 121 +
.../EnforceNativeOutputCollectorDelegator.java | 51 +
.../nativetask/testutil/MockValueClass.java | 80 +
.../nativetask/testutil/ResultVerifier.java | 162 +
.../testutil/ScenarioConfiguration.java | 58 +
.../nativetask/testutil/TestConstants.java | 71 +
.../mapred/nativetask/testutil/TestInput.java | 61 +
.../mapred/nativetask/utils/TestBytesUtil.java | 69 +
.../nativetask/utils/TestReadWriteBuffer.java | 62 +
.../nativetask/utils/TestSizedWritable.java | 34 +
.../src/test/resources/common_conf.xml | 48 +
.../src/test/resources/kvtest-conf.xml | 85 +
.../src/test/resources/log4j.properties | 19 +
.../src/test/resources/native_conf.xml | 30 +
.../src/test/resources/normal_conf.xml | 26 +
.../src/test/resources/test-combiner-conf.xml | 30 +
.../src/test/resources/test-compress-conf.xml | 50 +
.../src/test/resources/test-nonsort-conf.xml | 30 +
.../hadoop-mapreduce-client/pom.xml | 1 +
hadoop-mapreduce-project/pom.xml | 30 +-
hadoop-yarn-project/CHANGES.txt | 11 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 12 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 80 +-
.../timeline/webapp/CrossOriginFilter.java | 12 +-
.../timeline/webapp/TestCrossOriginFilter.java | 49 +
.../resourcemanager/recovery/RMStateStore.java | 3 +-
.../server/resourcemanager/TestRMRestart.java | 43 +
234 files changed, 53113 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
[25/50] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc
warnings. Contributed by Manu Zhang.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h
deleted file mode 100644
index b467822..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h
+++ /dev/null
@@ -1,19541 +0,0 @@
-// Copyright 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: wan@google.com (Zhanyong Wan)
-//
-// The Google C++ Testing Framework (Google Test)
-//
-// This header file defines the public API for Google Test. It should be
-// included by any test program that uses Google Test.
-//
-// IMPORTANT NOTE: Due to limitation of the C++ language, we have to
-// leave some internal implementation details in this header file.
-// They are clearly marked by comments like this:
-//
-// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM.
-//
-// Such code is NOT meant to be used by a user directly, and is subject
-// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user
-// program!
-//
-// Acknowledgment: Google Test borrowed the idea of automatic test
-// registration from Barthelemy Dagenais' (barthelemy@prologique.com)
-// easyUnit framework.
-
-#ifndef GTEST_INCLUDE_GTEST_GTEST_H_
-#define GTEST_INCLUDE_GTEST_GTEST_H_
-
-#include <limits>
-#include <vector>
-
-// Copyright 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee)
-//
-// The Google C++ Testing Framework (Google Test)
-//
-// This header file declares functions and macros used internally by
-// Google Test. They are subject to change without notice.
-
-#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_
-#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_
-
-// Copyright 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Authors: wan@google.com (Zhanyong Wan)
-//
-// Low-level types and utilities for porting Google Test to various
-// platforms. They are subject to change without notice. DO NOT USE
-// THEM IN USER CODE.
-
-#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
-#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
-
-// The user can define the following macros in the build script to
-// control Google Test's behavior. If the user doesn't define a macro
-// in this list, Google Test will define it.
-//
-// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2)
-// is/isn't available.
-// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions
-// are enabled.
-// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string
-// is/isn't available (some systems define
-// ::string, which is different to std::string).
-// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::string
-// is/isn't available (some systems define
-// ::wstring, which is different to std::wstring).
-// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular
-// expressions are/aren't available.
-// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that <pthread.h>
-// is/isn't available.
-// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't
-// enabled.
-// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that
-// std::wstring does/doesn't work (Google Test can
-// be used where std::wstring is unavailable).
-// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple
-// is/isn't available.
-// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the
-// compiler supports Microsoft's "Structured
-// Exception Handling".
-// GTEST_HAS_STREAM_REDIRECTION
-// - Define it to 1/0 to indicate whether the
-// platform supports I/O stream redirection using
-// dup() and dup2().
-// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google
-// Test's own tr1 tuple implementation should be
-// used. Unused when the user sets
-// GTEST_HAS_TR1_TUPLE to 0.
-// GTEST_LINKED_AS_SHARED_LIBRARY
-// - Define to 1 when compiling tests that use
-// Google Test as a shared library (known as
-// DLL on Windows).
-// GTEST_CREATE_SHARED_LIBRARY
-// - Define to 1 when compiling Google Test itself
-// as a shared library.
-
-// This header defines the following utilities:
-//
-// Macros indicating the current platform (defined to 1 if compiled on
-// the given platform; otherwise undefined):
-// GTEST_OS_AIX - IBM AIX
-// GTEST_OS_CYGWIN - Cygwin
-// GTEST_OS_HPUX - HP-UX
-// GTEST_OS_LINUX - Linux
-// GTEST_OS_LINUX_ANDROID - Google Android
-// GTEST_OS_MAC - Mac OS X
-// GTEST_OS_NACL - Google Native Client (NaCl)
-// GTEST_OS_SOLARIS - Sun Solaris
-// GTEST_OS_SYMBIAN - Symbian
-// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile)
-// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop
-// GTEST_OS_WINDOWS_MINGW - MinGW
-// GTEST_OS_WINDOWS_MOBILE - Windows Mobile
-// GTEST_OS_ZOS - z/OS
-//
-// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the
-// most stable support. Since core members of the Google Test project
-// don't have access to other platforms, support for them may be less
-// stable. If you notice any problems on your platform, please notify
-// googletestframework@googlegroups.com (patches for fixing them are
-// even more welcome!).
-//
-// Note that it is possible that none of the GTEST_OS_* macros are defined.
-//
-// Macros indicating available Google Test features (defined to 1 if
-// the corresponding feature is supported; otherwise undefined):
-// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized
-// tests)
-// GTEST_HAS_DEATH_TEST - death tests
-// GTEST_HAS_PARAM_TEST - value-parameterized tests
-// GTEST_HAS_TYPED_TEST - typed tests
-// GTEST_HAS_TYPED_TEST_P - type-parameterized tests
-// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with
-// GTEST_HAS_POSIX_RE (see above) which users can
-// define themselves.
-// GTEST_USES_SIMPLE_RE - our own simple regex is used;
-// the above two are mutually exclusive.
-// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ().
-//
-// Macros for basic C++ coding:
-// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning.
-// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a
-// variable don't have to be used.
-// GTEST_DISALLOW_ASSIGN_ - disables operator=.
-// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=.
-// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used.
-//
-// Synchronization:
-// Mutex, MutexLock, ThreadLocal, GetThreadCount()
-// - synchronization primitives.
-// GTEST_IS_THREADSAFE - defined to 1 to indicate that the above
-// synchronization primitives have real implementations
-// and Google Test is thread-safe; or 0 otherwise.
-//
-// Template meta programming:
-// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only.
-// IteratorTraits - partial implementation of std::iterator_traits, which
-// is not available in libCstd when compiled with Sun C++.
-//
-// Smart pointers:
-// scoped_ptr - as in TR2.
-//
-// Regular expressions:
-// RE - a simple regular expression class using the POSIX
-// Extended Regular Expression syntax on UNIX-like
-// platforms, or a reduced regular exception syntax on
-// other platforms, including Windows.
-//
-// Logging:
-// GTEST_LOG_() - logs messages at the specified severity level.
-// LogToStderr() - directs all log messages to stderr.
-// FlushInfoLog() - flushes informational log messages.
-//
-// Stdout and stderr capturing:
-// CaptureStdout() - starts capturing stdout.
-// GetCapturedStdout() - stops capturing stdout and returns the captured
-// string.
-// CaptureStderr() - starts capturing stderr.
-// GetCapturedStderr() - stops capturing stderr and returns the captured
-// string.
-//
-// Integer types:
-// TypeWithSize - maps an integer to a int type.
-// Int32, UInt32, Int64, UInt64, TimeInMillis
-// - integers of known sizes.
-// BiggestInt - the biggest signed integer type.
-//
-// Command-line utilities:
-// GTEST_FLAG() - references a flag.
-// GTEST_DECLARE_*() - declares a flag.
-// GTEST_DEFINE_*() - defines a flag.
-// GetArgvs() - returns the command line as a vector of strings.
-//
-// Environment variable utilities:
-// GetEnv() - gets the value of an environment variable.
-// BoolFromGTestEnv() - parses a bool environment variable.
-// Int32FromGTestEnv() - parses an Int32 environment variable.
-// StringFromGTestEnv() - parses a string environment variable.
-
-#include <ctype.h> // for isspace, etc
-#include <stddef.h> // for ptrdiff_t
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#ifndef _WIN32_WCE
-# include <sys/types.h>
-# include <sys/stat.h>
-#endif // !_WIN32_WCE
-
-#include <iostream> // NOLINT
-#include <sstream> // NOLINT
-#include <string> // NOLINT
-
-#define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com"
-#define GTEST_FLAG_PREFIX_ "gtest_"
-#define GTEST_FLAG_PREFIX_DASH_ "gtest-"
-#define GTEST_FLAG_PREFIX_UPPER_ "GTEST_"
-#define GTEST_NAME_ "Google Test"
-#define GTEST_PROJECT_URL_ "http://code.google.com/p/googletest/"
-
-// Determines the version of gcc that is used to compile this.
-#ifdef __GNUC__
-// 40302 means version 4.3.2.
-# define GTEST_GCC_VER_ \
- (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__)
-#endif // __GNUC__
-
-// Determines the platform on which Google Test is compiled.
-#ifdef __CYGWIN__
-# define GTEST_OS_CYGWIN 1
-#elif defined __SYMBIAN32__
-# define GTEST_OS_SYMBIAN 1
-#elif defined _WIN32
-# define GTEST_OS_WINDOWS 1
-# ifdef _WIN32_WCE
-# define GTEST_OS_WINDOWS_MOBILE 1
-# elif defined(__MINGW__) || defined(__MINGW32__)
-# define GTEST_OS_WINDOWS_MINGW 1
-# else
-# define GTEST_OS_WINDOWS_DESKTOP 1
-# endif // _WIN32_WCE
-#elif defined __APPLE__
-# define GTEST_OS_MAC 1
-#elif defined __linux__
-# define GTEST_OS_LINUX 1
-# ifdef ANDROID
-# define GTEST_OS_LINUX_ANDROID 1
-# endif // ANDROID
-#elif defined __MVS__
-# define GTEST_OS_ZOS 1
-#elif defined(__sun) && defined(__SVR4)
-# define GTEST_OS_SOLARIS 1
-#elif defined(_AIX)
-# define GTEST_OS_AIX 1
-#elif defined(__hpux)
-# define GTEST_OS_HPUX 1
-#elif defined __native_client__
-# define GTEST_OS_NACL 1
-#endif // __CYGWIN__
-
-// Brings in definitions for functions used in the testing::internal::posix
-// namespace (read, write, close, chdir, isatty, stat). We do not currently
-// use them on Windows Mobile.
-#if !GTEST_OS_WINDOWS
-// This assumes that non-Windows OSes provide unistd.h. For OSes where this
-// is not the case, we need to include headers that provide the functions
-// mentioned above.
-# include <unistd.h>
-# if !GTEST_OS_NACL
-// TODO(vladl@google.com): Remove this condition when Native Client SDK adds
-// strings.h (tracked in
-// http://code.google.com/p/nativeclient/issues/detail?id=1175).
-# include <strings.h> // Native Client doesn't provide strings.h.
-# endif
-#elif !GTEST_OS_WINDOWS_MOBILE
-# include <direct.h>
-# include <io.h>
-#endif
-
-// Defines this to true iff Google Test can use POSIX regular expressions.
-#ifndef GTEST_HAS_POSIX_RE
-# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS)
-#endif
-
-#if GTEST_HAS_POSIX_RE
-
-// On some platforms, <regex.h> needs someone to define size_t, and
-// won't compile otherwise. We can #include it here as we already
-// included <stdlib.h>, which is guaranteed to define size_t through
-// <stddef.h>.
-# include <regex.h> // NOLINT
-
-# define GTEST_USES_POSIX_RE 1
-
-#elif GTEST_OS_WINDOWS
-
-// <regex.h> is not available on Windows. Use our own simple regex
-// implementation instead.
-# define GTEST_USES_SIMPLE_RE 1
-
-#else
-
-// <regex.h> may not be available on this platform. Use our own
-// simple regex implementation instead.
-# define GTEST_USES_SIMPLE_RE 1
-
-#endif // GTEST_HAS_POSIX_RE
-
-#ifndef GTEST_HAS_EXCEPTIONS
-// The user didn't tell us whether exceptions are enabled, so we need
-// to figure it out.
-# if defined(_MSC_VER) || defined(__BORLANDC__)
-// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS
-// macro to enable exceptions, so we'll do the same.
-// Assumes that exceptions are enabled by default.
-# ifndef _HAS_EXCEPTIONS
-# define _HAS_EXCEPTIONS 1
-# endif // _HAS_EXCEPTIONS
-# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS
-# elif defined(__GNUC__) && __EXCEPTIONS
-// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled.
-# define GTEST_HAS_EXCEPTIONS 1
-# elif defined(__SUNPRO_CC)
-// Sun Pro CC supports exceptions. However, there is no compile-time way of
-// detecting whether they are enabled or not. Therefore, we assume that
-// they are enabled unless the user tells us otherwise.
-# define GTEST_HAS_EXCEPTIONS 1
-# elif defined(__IBMCPP__) && __EXCEPTIONS
-// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled.
-# define GTEST_HAS_EXCEPTIONS 1
-# elif defined(__HP_aCC)
-// Exception handling is in effect by default in HP aCC compiler. It has to
-// be turned of by +noeh compiler option if desired.
-# define GTEST_HAS_EXCEPTIONS 1
-# else
-// For other compilers, we assume exceptions are disabled to be
-// conservative.
-# define GTEST_HAS_EXCEPTIONS 0
-# endif // defined(_MSC_VER) || defined(__BORLANDC__)
-#endif // GTEST_HAS_EXCEPTIONS
-
-#if !defined(GTEST_HAS_STD_STRING)
-// Even though we don't use this macro any longer, we keep it in case
-// some clients still depend on it.
-# define GTEST_HAS_STD_STRING 1
-#elif !GTEST_HAS_STD_STRING
-// The user told us that ::std::string isn't available.
-# error "Google Test cannot be used where ::std::string isn't available."
-#endif // !defined(GTEST_HAS_STD_STRING)
-
-#ifndef GTEST_HAS_GLOBAL_STRING
-// The user didn't tell us whether ::string is available, so we need
-// to figure it out.
-
-# define GTEST_HAS_GLOBAL_STRING 0
-
-#endif // GTEST_HAS_GLOBAL_STRING
-
-#ifndef GTEST_HAS_STD_WSTRING
-// The user didn't tell us whether ::std::wstring is available, so we need
-// to figure it out.
-// TODO(wan@google.com): uses autoconf to detect whether ::std::wstring
-// is available.
-
-// Cygwin 1.7 and below doesn't support ::std::wstring.
-// Solaris' libc++ doesn't support it either. Android has
-// no support for it at least as recent as Froyo (2.2).
-# define GTEST_HAS_STD_WSTRING \
- (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS))
-
-#endif // GTEST_HAS_STD_WSTRING
-
-#ifndef GTEST_HAS_GLOBAL_WSTRING
-// The user didn't tell us whether ::wstring is available, so we need
-// to figure it out.
-# define GTEST_HAS_GLOBAL_WSTRING \
- (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING)
-#endif // GTEST_HAS_GLOBAL_WSTRING
-
-// Determines whether RTTI is available.
-#ifndef GTEST_HAS_RTTI
-// The user didn't tell us whether RTTI is enabled, so we need to
-// figure it out.
-
-# ifdef _MSC_VER
-
-# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled.
-# define GTEST_HAS_RTTI 1
-# else
-# define GTEST_HAS_RTTI 0
-# endif
-
-// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled.
-# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302)
-
-# ifdef __GXX_RTTI
-# define GTEST_HAS_RTTI 1
-# else
-# define GTEST_HAS_RTTI 0
-# endif // __GXX_RTTI
-
-// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if
-// both the typeid and dynamic_cast features are present.
-# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900)
-
-# ifdef __RTTI_ALL__
-# define GTEST_HAS_RTTI 1
-# else
-# define GTEST_HAS_RTTI 0
-# endif
-
-# else
-
-// For all other compilers, we assume RTTI is enabled.
-# define GTEST_HAS_RTTI 1
-
-# endif // _MSC_VER
-
-#endif // GTEST_HAS_RTTI
-
-// It's this header's responsibility to #include <typeinfo> when RTTI
-// is enabled.
-#if GTEST_HAS_RTTI
-# include <typeinfo>
-#endif
-
-// Determines whether Google Test can use the pthreads library.
-#ifndef GTEST_HAS_PTHREAD
-// The user didn't tell us explicitly, so we assume pthreads support is
-// available on Linux and Mac.
-//
-// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0
-// to your compiler flags.
-# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX)
-#endif // GTEST_HAS_PTHREAD
-
-#if GTEST_HAS_PTHREAD
-// gtest-port.h guarantees to #include <pthread.h> when GTEST_HAS_PTHREAD is
-// true.
-# include <pthread.h> // NOLINT
-
-// For timespec and nanosleep, used below.
-# include <time.h> // NOLINT
-#endif
-
-// Determines whether Google Test can use tr1/tuple. You can define
-// this macro to 0 to prevent Google Test from using tuple (any
-// feature depending on tuple with be disabled in this mode).
-#ifndef GTEST_HAS_TR1_TUPLE
-// The user didn't tell us not to do it, so we assume it's OK.
-# define GTEST_HAS_TR1_TUPLE 1
-#endif // GTEST_HAS_TR1_TUPLE
-
-// Determines whether Google Test's own tr1 tuple implementation
-// should be used.
-#ifndef GTEST_USE_OWN_TR1_TUPLE
-// The user didn't tell us, so we need to figure it out.
-
-// We use our own TR1 tuple if we aren't sure the user has an
-// implementation of it already. At this time, GCC 4.0.0+ and MSVC
-// 2010 are the only mainstream compilers that come with a TR1 tuple
-// implementation. NVIDIA's CUDA NVCC compiler pretends to be GCC by
-// defining __GNUC__ and friends, but cannot compile GCC's tuple
-// implementation. MSVC 2008 (9.0) provides TR1 tuple in a 323 MB
-// Feature Pack download, which we cannot assume the user has.
-# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000)) \
- || _MSC_VER >= 1600
-# define GTEST_USE_OWN_TR1_TUPLE 0
-# else
-# define GTEST_USE_OWN_TR1_TUPLE 1
-# endif
-
-#endif // GTEST_USE_OWN_TR1_TUPLE
-
-// To avoid conditional compilation everywhere, we make it
-// gtest-port.h's responsibility to #include the header implementing
-// tr1/tuple.
-#if GTEST_HAS_TR1_TUPLE
-
-# if GTEST_USE_OWN_TR1_TUPLE
-// This file was GENERATED by a script. DO NOT EDIT BY HAND!!!
-
-// Copyright 2009 Google Inc.
-// All Rights Reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: wan@google.com (Zhanyong Wan)
-
-// Implements a subset of TR1 tuple needed by Google Test and Google Mock.
-
-#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
-#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
-
-#include <utility> // For ::std::pair.
-
-// The compiler used in Symbian has a bug that prevents us from declaring the
-// tuple template as a friend (it complains that tuple is redefined). This
-// hack bypasses the bug by declaring the members that should otherwise be
-// private as public.
-// Sun Studio versions < 12 also have the above bug.
-#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590)
-# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public:
-#else
-# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \
- template <GTEST_10_TYPENAMES_(U)> friend class tuple; \
- private:
-#endif
-
-// GTEST_n_TUPLE_(T) is the type of an n-tuple.
-#define GTEST_0_TUPLE_(T) tuple<>
-#define GTEST_1_TUPLE_(T) tuple<T##0, void, void, void, void, void, void, \
- void, void, void>
-#define GTEST_2_TUPLE_(T) tuple<T##0, T##1, void, void, void, void, void, \
- void, void, void>
-#define GTEST_3_TUPLE_(T) tuple<T##0, T##1, T##2, void, void, void, void, \
- void, void, void>
-#define GTEST_4_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, void, void, void, \
- void, void, void>
-#define GTEST_5_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, void, void, \
- void, void, void>
-#define GTEST_6_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, void, \
- void, void, void>
-#define GTEST_7_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
- void, void, void>
-#define GTEST_8_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
- T##7, void, void>
-#define GTEST_9_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
- T##7, T##8, void>
-#define GTEST_10_TUPLE_(T) tuple<T##0, T##1, T##2, T##3, T##4, T##5, T##6, \
- T##7, T##8, T##9>
-
-// GTEST_n_TYPENAMES_(T) declares a list of n typenames.
-#define GTEST_0_TYPENAMES_(T)
-#define GTEST_1_TYPENAMES_(T) typename T##0
-#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1
-#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2
-#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3
-#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4
-#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4, typename T##5
-#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4, typename T##5, typename T##6
-#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4, typename T##5, typename T##6, typename T##7
-#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4, typename T##5, typename T##6, \
- typename T##7, typename T##8
-#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \
- typename T##3, typename T##4, typename T##5, typename T##6, \
- typename T##7, typename T##8, typename T##9
-
-// In theory, defining stuff in the ::std namespace is undefined
-// behavior. We can do this as we are playing the role of a standard
-// library vendor.
-namespace std {
-namespace tr1 {
-
-template <typename T0 = void, typename T1 = void, typename T2 = void,
- typename T3 = void, typename T4 = void, typename T5 = void,
- typename T6 = void, typename T7 = void, typename T8 = void,
- typename T9 = void>
-class tuple;
-
-// Anything in namespace gtest_internal is Google Test's INTERNAL
-// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code.
-namespace gtest_internal {
-
-// ByRef<T>::type is T if T is a reference; otherwise it's const T&.
-template <typename T>
-struct ByRef { typedef const T& type; }; // NOLINT
-template <typename T>
-struct ByRef<T&> { typedef T& type; }; // NOLINT
-
-// A handy wrapper for ByRef.
-#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef<T>::type
-
-// AddRef<T>::type is T if T is a reference; otherwise it's T&. This
-// is the same as tr1::add_reference<T>::type.
-template <typename T>
-struct AddRef { typedef T& type; }; // NOLINT
-template <typename T>
-struct AddRef<T&> { typedef T& type; }; // NOLINT
-
-// A handy wrapper for AddRef.
-#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef<T>::type
-
-// A helper for implementing get<k>().
-template <int k> class Get;
-
-// A helper for implementing tuple_element<k, T>. kIndexValid is true
-// iff k < the number of fields in tuple type T.
-template <bool kIndexValid, int kIndex, class Tuple>
-struct TupleElement;
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 0, GTEST_10_TUPLE_(T)> { typedef T0 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 1, GTEST_10_TUPLE_(T)> { typedef T1 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 2, GTEST_10_TUPLE_(T)> { typedef T2 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 3, GTEST_10_TUPLE_(T)> { typedef T3 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 4, GTEST_10_TUPLE_(T)> { typedef T4 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 5, GTEST_10_TUPLE_(T)> { typedef T5 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 6, GTEST_10_TUPLE_(T)> { typedef T6 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 7, GTEST_10_TUPLE_(T)> { typedef T7 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 8, GTEST_10_TUPLE_(T)> { typedef T8 type; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct TupleElement<true, 9, GTEST_10_TUPLE_(T)> { typedef T9 type; };
-
-} // namespace gtest_internal
-
-template <>
-class tuple<> {
- public:
- tuple() {}
- tuple(const tuple& /* t */) {}
- tuple& operator=(const tuple& /* t */) { return *this; }
-};
-
-template <GTEST_1_TYPENAMES_(T)>
-class GTEST_1_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {}
-
- tuple(const tuple& t) : f0_(t.f0_) {}
-
- template <GTEST_1_TYPENAMES_(U)>
- tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_1_TYPENAMES_(U)>
- tuple& operator=(const GTEST_1_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_1_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) {
- f0_ = t.f0_;
- return *this;
- }
-
- T0 f0_;
-};
-
-template <GTEST_2_TYPENAMES_(T)>
-class GTEST_2_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0),
- f1_(f1) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {}
-
- template <GTEST_2_TYPENAMES_(U)>
- tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {}
- template <typename U0, typename U1>
- tuple(const ::std::pair<U0, U1>& p) : f0_(p.first), f1_(p.second) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_2_TYPENAMES_(U)>
- tuple& operator=(const GTEST_2_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
- template <typename U0, typename U1>
- tuple& operator=(const ::std::pair<U0, U1>& p) {
- f0_ = p.first;
- f1_ = p.second;
- return *this;
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_2_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
-};
-
-template <GTEST_3_TYPENAMES_(T)>
-class GTEST_3_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {}
-
- template <GTEST_3_TYPENAMES_(U)>
- tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_3_TYPENAMES_(U)>
- tuple& operator=(const GTEST_3_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_3_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
-};
-
-template <GTEST_4_TYPENAMES_(T)>
-class GTEST_4_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2),
- f3_(f3) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {}
-
- template <GTEST_4_TYPENAMES_(U)>
- tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_4_TYPENAMES_(U)>
- tuple& operator=(const GTEST_4_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_4_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
-};
-
-template <GTEST_5_TYPENAMES_(T)>
-class GTEST_5_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3,
- GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_) {}
-
- template <GTEST_5_TYPENAMES_(U)>
- tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_5_TYPENAMES_(U)>
- tuple& operator=(const GTEST_5_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_5_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
-};
-
-template <GTEST_6_TYPENAMES_(T)>
-class GTEST_6_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
- GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
- f5_(f5) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_), f5_(t.f5_) {}
-
- template <GTEST_6_TYPENAMES_(U)>
- tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_6_TYPENAMES_(U)>
- tuple& operator=(const GTEST_6_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_6_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- f5_ = t.f5_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
- T5 f5_;
-};
-
-template <GTEST_7_TYPENAMES_(T)>
-class GTEST_7_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
- GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2),
- f3_(f3), f4_(f4), f5_(f5), f6_(f6) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {}
-
- template <GTEST_7_TYPENAMES_(U)>
- tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_7_TYPENAMES_(U)>
- tuple& operator=(const GTEST_7_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_7_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- f5_ = t.f5_;
- f6_ = t.f6_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
- T5 f5_;
- T6 f6_;
-};
-
-template <GTEST_8_TYPENAMES_(T)>
-class GTEST_8_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
- GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6,
- GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
- f5_(f5), f6_(f6), f7_(f7) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {}
-
- template <GTEST_8_TYPENAMES_(U)>
- tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_8_TYPENAMES_(U)>
- tuple& operator=(const GTEST_8_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_8_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- f5_ = t.f5_;
- f6_ = t.f6_;
- f7_ = t.f7_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
- T5 f5_;
- T6 f6_;
- T7 f7_;
-};
-
-template <GTEST_9_TYPENAMES_(T)>
-class GTEST_9_TUPLE_(T) {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
- GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7,
- GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4),
- f5_(f5), f6_(f6), f7_(f7), f8_(f8) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {}
-
- template <GTEST_9_TYPENAMES_(U)>
- tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_9_TYPENAMES_(U)>
- tuple& operator=(const GTEST_9_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_9_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- f5_ = t.f5_;
- f6_ = t.f6_;
- f7_ = t.f7_;
- f8_ = t.f8_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
- T5 f5_;
- T6 f6_;
- T7 f7_;
- T8 f8_;
-};
-
-template <GTEST_10_TYPENAMES_(T)>
-class tuple {
- public:
- template <int k> friend class gtest_internal::Get;
-
- tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(),
- f9_() {}
-
- explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1,
- GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4,
- GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7,
- GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2),
- f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {}
-
- tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_),
- f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {}
-
- template <GTEST_10_TYPENAMES_(U)>
- tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_),
- f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_),
- f9_(t.f9_) {}
-
- tuple& operator=(const tuple& t) { return CopyFrom(t); }
-
- template <GTEST_10_TYPENAMES_(U)>
- tuple& operator=(const GTEST_10_TUPLE_(U)& t) {
- return CopyFrom(t);
- }
-
- GTEST_DECLARE_TUPLE_AS_FRIEND_
-
- template <GTEST_10_TYPENAMES_(U)>
- tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) {
- f0_ = t.f0_;
- f1_ = t.f1_;
- f2_ = t.f2_;
- f3_ = t.f3_;
- f4_ = t.f4_;
- f5_ = t.f5_;
- f6_ = t.f6_;
- f7_ = t.f7_;
- f8_ = t.f8_;
- f9_ = t.f9_;
- return *this;
- }
-
- T0 f0_;
- T1 f1_;
- T2 f2_;
- T3 f3_;
- T4 f4_;
- T5 f5_;
- T6 f6_;
- T7 f7_;
- T8 f8_;
- T9 f9_;
-};
-
-// 6.1.3.2 Tuple creation functions.
-
-// Known limitations: we don't support passing an
-// std::tr1::reference_wrapper<T> to make_tuple(). And we don't
-// implement tie().
-
-inline tuple<> make_tuple() { return tuple<>(); }
-
-template <GTEST_1_TYPENAMES_(T)>
-inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) {
- return GTEST_1_TUPLE_(T)(f0);
-}
-
-template <GTEST_2_TYPENAMES_(T)>
-inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) {
- return GTEST_2_TUPLE_(T)(f0, f1);
-}
-
-template <GTEST_3_TYPENAMES_(T)>
-inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) {
- return GTEST_3_TUPLE_(T)(f0, f1, f2);
-}
-
-template <GTEST_4_TYPENAMES_(T)>
-inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3) {
- return GTEST_4_TUPLE_(T)(f0, f1, f2, f3);
-}
-
-template <GTEST_5_TYPENAMES_(T)>
-inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4) {
- return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4);
-}
-
-template <GTEST_6_TYPENAMES_(T)>
-inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4, const T5& f5) {
- return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5);
-}
-
-template <GTEST_7_TYPENAMES_(T)>
-inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4, const T5& f5, const T6& f6) {
- return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6);
-}
-
-template <GTEST_8_TYPENAMES_(T)>
-inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) {
- return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7);
-}
-
-template <GTEST_9_TYPENAMES_(T)>
-inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7,
- const T8& f8) {
- return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8);
-}
-
-template <GTEST_10_TYPENAMES_(T)>
-inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2,
- const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7,
- const T8& f8, const T9& f9) {
- return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9);
-}
-
-// 6.1.3.3 Tuple helper classes.
-
-template <typename Tuple> struct tuple_size;
-
-template <GTEST_0_TYPENAMES_(T)>
-struct tuple_size<GTEST_0_TUPLE_(T)> { static const int value = 0; };
-
-template <GTEST_1_TYPENAMES_(T)>
-struct tuple_size<GTEST_1_TUPLE_(T)> { static const int value = 1; };
-
-template <GTEST_2_TYPENAMES_(T)>
-struct tuple_size<GTEST_2_TUPLE_(T)> { static const int value = 2; };
-
-template <GTEST_3_TYPENAMES_(T)>
-struct tuple_size<GTEST_3_TUPLE_(T)> { static const int value = 3; };
-
-template <GTEST_4_TYPENAMES_(T)>
-struct tuple_size<GTEST_4_TUPLE_(T)> { static const int value = 4; };
-
-template <GTEST_5_TYPENAMES_(T)>
-struct tuple_size<GTEST_5_TUPLE_(T)> { static const int value = 5; };
-
-template <GTEST_6_TYPENAMES_(T)>
-struct tuple_size<GTEST_6_TUPLE_(T)> { static const int value = 6; };
-
-template <GTEST_7_TYPENAMES_(T)>
-struct tuple_size<GTEST_7_TUPLE_(T)> { static const int value = 7; };
-
-template <GTEST_8_TYPENAMES_(T)>
-struct tuple_size<GTEST_8_TUPLE_(T)> { static const int value = 8; };
-
-template <GTEST_9_TYPENAMES_(T)>
-struct tuple_size<GTEST_9_TUPLE_(T)> { static const int value = 9; };
-
-template <GTEST_10_TYPENAMES_(T)>
-struct tuple_size<GTEST_10_TUPLE_(T)> { static const int value = 10; };
-
-template <int k, class Tuple>
-struct tuple_element {
- typedef typename gtest_internal::TupleElement<
- k < (tuple_size<Tuple>::value), k, Tuple>::type type;
-};
-
-#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element<k, Tuple >::type
-
-// 6.1.3.4 Element access.
-
-namespace gtest_internal {
-
-template <>
-class Get<0> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple))
- Field(Tuple& t) { return t.f0_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple))
- ConstField(const Tuple& t) { return t.f0_; }
-};
-
-template <>
-class Get<1> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple))
- Field(Tuple& t) { return t.f1_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple))
- ConstField(const Tuple& t) { return t.f1_; }
-};
-
-template <>
-class Get<2> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple))
- Field(Tuple& t) { return t.f2_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple))
- ConstField(const Tuple& t) { return t.f2_; }
-};
-
-template <>
-class Get<3> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple))
- Field(Tuple& t) { return t.f3_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple))
- ConstField(const Tuple& t) { return t.f3_; }
-};
-
-template <>
-class Get<4> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple))
- Field(Tuple& t) { return t.f4_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple))
- ConstField(const Tuple& t) { return t.f4_; }
-};
-
-template <>
-class Get<5> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple))
- Field(Tuple& t) { return t.f5_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple))
- ConstField(const Tuple& t) { return t.f5_; }
-};
-
-template <>
-class Get<6> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple))
- Field(Tuple& t) { return t.f6_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple))
- ConstField(const Tuple& t) { return t.f6_; }
-};
-
-template <>
-class Get<7> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple))
- Field(Tuple& t) { return t.f7_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple))
- ConstField(const Tuple& t) { return t.f7_; }
-};
-
-template <>
-class Get<8> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple))
- Field(Tuple& t) { return t.f8_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple))
- ConstField(const Tuple& t) { return t.f8_; }
-};
-
-template <>
-class Get<9> {
- public:
- template <class Tuple>
- static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple))
- Field(Tuple& t) { return t.f9_; } // NOLINT
-
- template <class Tuple>
- static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple))
- ConstField(const Tuple& t) { return t.f9_; }
-};
-
-} // namespace gtest_internal
-
-template <int k, GTEST_10_TYPENAMES_(T)>
-GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T)))
-get(GTEST_10_TUPLE_(T)& t) {
- return gtest_internal::Get<k>::Field(t);
-}
-
-template <int k, GTEST_10_TYPENAMES_(T)>
-GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T)))
-get(const GTEST_10_TUPLE_(T)& t) {
- return gtest_internal::Get<k>::ConstField(t);
-}
-
-// 6.1.3.5 Relational operators
-
-// We only implement == and !=, as we don't have a need for the rest yet.
-
-namespace gtest_internal {
-
-// SameSizeTuplePrefixComparator<k, k>::Eq(t1, t2) returns true if the
-// first k fields of t1 equals the first k fields of t2.
-// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if
-// k1 != k2.
-template <int kSize1, int kSize2>
-struct SameSizeTuplePrefixComparator;
-
-template <>
-struct SameSizeTuplePrefixComparator<0, 0> {
- template <class Tuple1, class Tuple2>
- static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) {
- return true;
- }
-};
-
-template <int k>
-struct SameSizeTuplePrefixComparator<k, k> {
- template <class Tuple1, class Tuple2>
- static bool Eq(const Tuple1& t1, const Tuple2& t2) {
- return SameSizeTuplePrefixComparator<k - 1, k - 1>::Eq(t1, t2) &&
- ::std::tr1::get<k - 1>(t1) == ::std::tr1::get<k - 1>(t2);
- }
-};
-
-} // namespace gtest_internal
-
-template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)>
-inline bool operator==(const GTEST_10_TUPLE_(T)& t,
- const GTEST_10_TUPLE_(U)& u) {
- return gtest_internal::SameSizeTuplePrefixComparator<
- tuple_size<GTEST_10_TUPLE_(T)>::value,
- tuple_size<GTEST_10_TUPLE_(U)>::value>::Eq(t, u);
-}
-
-template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)>
-inline bool operator!=(const GTEST_10_TUPLE_(T)& t,
- const GTEST_10_TUPLE_(U)& u) { return !(t == u); }
-
-// 6.1.4 Pairs.
-// Unimplemented.
-
-} // namespace tr1
-} // namespace std
-
-#undef GTEST_0_TUPLE_
-#undef GTEST_1_TUPLE_
-#undef GTEST_2_TUPLE_
-#undef GTEST_3_TUPLE_
-#undef GTEST_4_TUPLE_
-#undef GTEST_5_TUPLE_
-#undef GTEST_6_TUPLE_
-#undef GTEST_7_TUPLE_
-#undef GTEST_8_TUPLE_
-#undef GTEST_9_TUPLE_
-#undef GTEST_10_TUPLE_
-
-#undef GTEST_0_TYPENAMES_
-#undef GTEST_1_TYPENAMES_
-#undef GTEST_2_TYPENAMES_
-#undef GTEST_3_TYPENAMES_
-#undef GTEST_4_TYPENAMES_
-#undef GTEST_5_TYPENAMES_
-#undef GTEST_6_TYPENAMES_
-#undef GTEST_7_TYPENAMES_
-#undef GTEST_8_TYPENAMES_
-#undef GTEST_9_TYPENAMES_
-#undef GTEST_10_TYPENAMES_
-
-#undef GTEST_DECLARE_TUPLE_AS_FRIEND_
-#undef GTEST_BY_REF_
-#undef GTEST_ADD_REF_
-#undef GTEST_TUPLE_ELEMENT_
-
-#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_
-# elif GTEST_OS_SYMBIAN
-
-// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to
-// use STLport's tuple implementation, which unfortunately doesn't
-// work as the copy of STLport distributed with Symbian is incomplete.
-// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to
-// use its own tuple implementation.
-# ifdef BOOST_HAS_TR1_TUPLE
-# undef BOOST_HAS_TR1_TUPLE
-# endif // BOOST_HAS_TR1_TUPLE
-
-// This prevents <boost/tr1/detail/config.hpp>, which defines
-// BOOST_HAS_TR1_TUPLE, from being #included by Boost's <tuple>.
-# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED
-# include <tuple>
-
-# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000)
-// GCC 4.0+ implements tr1/tuple in the <tr1/tuple> header. This does
-// not conform to the TR1 spec, which requires the header to be <tuple>.
-
-# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302
-// Until version 4.3.2, gcc has a bug that causes <tr1/functional>,
-// which is #included by <tr1/tuple>, to not compile when RTTI is
-// disabled. _TR1_FUNCTIONAL is the header guard for
-// <tr1/functional>. Hence the following #define is a hack to prevent
-// <tr1/functional> from being included.
-# define _TR1_FUNCTIONAL 1
-# include <tr1/tuple>
-# undef _TR1_FUNCTIONAL // Allows the user to #include
- // <tr1/functional> if he chooses to.
-# else
-# include <tr1/tuple> // NOLINT
-# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302
-
-# else
-// If the compiler is not GCC 4.0+, we assume the user is using a
-// spec-conforming TR1 implementation.
-# include <tuple> // NOLINT
-# endif // GTEST_USE_OWN_TR1_TUPLE
-
-#endif // GTEST_HAS_TR1_TUPLE
-
-// Determines whether clone(2) is supported.
-// Usually it will only be available on Linux, excluding
-// Linux on the Itanium architecture.
-// Also see http://linux.die.net/man/2/clone.
-#ifndef GTEST_HAS_CLONE
-// The user didn't tell us, so we need to figure it out.
-
-# if GTEST_OS_LINUX && !defined(__ia64__)
-# define GTEST_HAS_CLONE 1
-# else
-# define GTEST_HAS_CLONE 0
-# endif // GTEST_OS_LINUX && !defined(__ia64__)
-
-#endif // GTEST_HAS_CLONE
-
-// Determines whether to support stream redirection. This is used to test
-// output correctness and to implement death tests.
-#ifndef GTEST_HAS_STREAM_REDIRECTION
-// By default, we assume that stream redirection is supported on all
-// platforms except known mobile ones.
-# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN
-# define GTEST_HAS_STREAM_REDIRECTION 0
-# else
-# define GTEST_HAS_STREAM_REDIRECTION 1
-# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN
-#endif // GTEST_HAS_STREAM_REDIRECTION
-
-// Determines whether to support death tests.
-// Google Test does not support death tests for VC 7.1 and earlier as
-// abort() in a VC 7.1 application compiled as GUI in debug config
-// pops up a dialog window that cannot be suppressed programmatically.
-#if (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \
- (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \
- GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX)
-# define GTEST_HAS_DEATH_TEST 1
-# include <vector> // NOLINT
-#endif
-
-// We don't support MSVC 7.1 with exceptions disabled now. Therefore
-// all the compilers we care about are adequate for supporting
-// value-parameterized tests.
-#define GTEST_HAS_PARAM_TEST 1
-
-// Determines whether to support type-driven tests.
-
-// Typed tests need <typeinfo> and variadic macros, which GCC, VC++ 8.0,
-// Sun Pro CC, IBM Visual Age, and HP aCC support.
-#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \
- defined(__IBMCPP__) || defined(__HP_aCC)
-# define GTEST_HAS_TYPED_TEST 1
-# define GTEST_HAS_TYPED_TEST_P 1
-#endif
-
-// Determines whether to support Combine(). This only makes sense when
-// value-parameterized tests are enabled. The implementation doesn't
-// work on Sun Studio since it doesn't understand templated conversion
-// operators.
-#if GTEST_HAS_PARAM_TEST && GTEST_HAS_TR1_TUPLE && !defined(__SUNPRO_CC)
-# define GTEST_HAS_COMBINE 1
-#endif
-
-// Determines whether the system compiler uses UTF-16 for encoding wide strings.
-#define GTEST_WIDE_STRING_USES_UTF16_ \
- (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX)
-
-// Determines whether test results can be streamed to a socket.
-#if GTEST_OS_LINUX
-# define GTEST_CAN_STREAM_RESULTS_ 1
-#endif
-
-// Defines some utility macros.
-
-// The GNU compiler emits a warning if nested "if" statements are followed by
-// an "else" statement and braces are not used to explicitly disambiguate the
-// "else" binding. This leads to problems with code like:
-//
-// if (gate)
-// ASSERT_*(condition) << "Some message";
-//
-// The "switch (0) case 0:" idiom is used to suppress this.
-#ifdef __INTEL_COMPILER
-# define GTEST_AMBIGUOUS_ELSE_BLOCKER_
-#else
-# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT
-#endif
-
-// Use this annotation at the end of a struct/class definition to
-// prevent the compiler from optimizing away instances that are never
-// used. This is useful when all interesting logic happens inside the
-// c'tor and / or d'tor. Example:
-//
-// struct Foo {
-// Foo() { ... }
-// } GTEST_ATTRIBUTE_UNUSED_;
-//
-// Also use it after a variable or parameter declaration to tell the
-// compiler the variable/parameter does not have to be used.
-#if defined(__GNUC__) && !defined(COMPILER_ICC)
-# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused))
-#else
-# define GTEST_ATTRIBUTE_UNUSED_
-#endif
-
-// A macro to disallow operator=
-// This should be used in the private: declarations for a class.
-#define GTEST_DISALLOW_ASSIGN_(type)\
- void operator=(type const &)
-
-// A macro to disallow copy constructor and operator=
-// This should be used in the private: declarations for a class.
-#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type)\
- type(type const &);\
- GTEST_DISALLOW_ASSIGN_(type)
-
-// Tell the compiler to warn about unused return values for functions declared
-// with this macro. The macro should be used on function declarations
-// following the argument list:
-//
-// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_;
-#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC)
-# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result))
-#else
-# define GTEST_MUST_USE_RESULT_
-#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC
-
-// Determine whether the compiler supports Microsoft's Structured Exception
-// Handling. This is supported by several Windows compilers but generally
-// does not exist on any other system.
-#ifndef GTEST_HAS_SEH
-// The user didn't tell us, so we need to figure it out.
-
-# if defined(_MSC_VER) || defined(__BORLANDC__)
-// These two compilers are known to support SEH.
-# define GTEST_HAS_SEH 1
-# else
-// Assume no SEH.
-# define GTEST_HAS_SEH 0
-# endif
-
-#endif // GTEST_HAS_SEH
-
-#ifdef _MSC_VER
-
-# if GTEST_LINKED_AS_SHARED_LIBRARY
-# define GTEST_API_ __declspec(dllimport)
-# elif GTEST_CREATE_SHARED_LIBRARY
-# define GTEST_API_ __declspec(dllexport)
-# endif
-
-#endif // _MSC_VER
-
-#ifndef GTEST_API_
-# define GTEST_API_
-#endif
-
-#ifdef __GNUC__
-// Ask the compiler to never inline a given function.
-# define GTEST_NO_INLINE_ __attribute__((noinline))
-#else
-# define GTEST_NO_INLINE_
-#endif
-
-namespace testing {
-
-class Message;
-
-namespace internal {
-
-class String;
-
-// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time
-// expression is true. For example, you could use it to verify the
-// size of a static array:
-//
-// GTEST_COMPILE_ASSERT_(ARRAYSIZE(content_type_names) == CONTENT_NUM_TYPES,
-// content_type_names_incorrect_size);
-//
-// or to make sure a struct is smaller than a certain size:
-//
-// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large);
-//
-// The second argument to the macro is the name of the variable. If
-// the expression is false, most compilers will issue a warning/error
-// containing the name of the variable.
-
-template <bool>
-struct CompileAssert {
-};
-
-#define GTEST_COMPILE_ASSERT_(expr, msg) \
- typedef ::testing::internal::CompileAssert<(bool(expr))> \
- msg[bool(expr) ? 1 : -1]
-
-// Implementation details of GTEST_COMPILE_ASSERT_:
-//
-// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1
-// elements (and thus is invalid) when the expression is false.
-//
-// - The simpler definition
-//
-// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1]
-//
-// does not work, as gcc supports variable-length arrays whose sizes
-// are determined at run-time (this is gcc's extension and not part
-// of the C++ standard). As a result, gcc fails to reject the
-// following code with the simple definition:
-//
-// int foo;
-// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is
-// // not a compile-time constant.
-//
-// - By using the type CompileAssert<(bool(expr))>, we ensures that
-// expr is a compile-time constant. (Template arguments must be
-// determined at compile-time.)
-//
-// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
-// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written
-//
-// CompileAssert<bool(expr)>
-//
-// instead, these compilers will refuse to compile
-//
-// GTEST_COMPILE_ASSERT_(5 > 0, some_message);
-//
-// (They seem to think the ">" in "5 > 0" marks the end of the
-// template argument list.)
-//
-// - The array size is (bool(expr) ? 1 : -1), instead of simply
-//
-// ((expr) ? 1 : -1).
-//
-// This is to avoid running into a bug in MS VC 7.1, which
-// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
-
-// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h.
-//
-// This template is declared, but intentionally undefined.
-template <typename T1, typename T2>
-struct StaticAssertTypeEqHelper;
-
-template <typename T>
-struct StaticAssertTypeEqHelper<T, T> {};
-
-#if GTEST_HAS_GLOBAL_STRING
-typedef ::string string;
-#else
-typedef ::std::string string;
-#endif // GTEST_HAS_GLOBAL_STRING
-
-#if GTEST_HAS_GLOBAL_WSTRING
-typedef ::wstring wstring;
-#elif GTEST_HAS_STD_WSTRING
-typedef ::std::wstring wstring;
-#endif // GTEST_HAS_GLOBAL_WSTRING
-
-// A helper for suppressing warnings on constant condition. It just
-// returns 'condition'.
-GTEST_API_ bool IsTrue(bool condition);
-
-// Defines scoped_ptr.
-
-// This implementation of scoped_ptr is PARTIAL - it only contains
-// enough stuff to satisfy Google Test's need.
-template <typename T>
-class scoped_ptr {
- public:
- typedef T element_type;
-
- explicit scoped_ptr(T* p = NULL) : ptr_(p) {}
- ~scoped_ptr() { reset(); }
-
- T& operator*() const { return *ptr_; }
- T* operator->() const { return ptr_; }
- T* get() const { return ptr_; }
-
- T* release() {
- T* const ptr = ptr_;
- ptr_ = NULL;
- return ptr;
- }
-
- void reset(T* p = NULL) {
- if (p != ptr_) {
- if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type.
- delete ptr_;
- }
- ptr_ = p;
- }
- }
- private:
- T* ptr_;
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr);
-};
-
-// Defines RE.
-
-// A simple C++ wrapper for <regex.h>. It uses the POSIX Extended
-// Regular Expression syntax.
-class GTEST_API_ RE {
- public:
- // A copy constructor is required by the Standard to initialize object
- // references from r-values.
- RE(const RE& other) { Init(other.pattern()); }
-
- // Constructs an RE from a string.
- RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT
-
-#if GTEST_HAS_GLOBAL_STRING
-
- RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT
-
-#endif // GTEST_HAS_GLOBAL_STRING
-
- RE(const char* regex) { Init(regex); } // NOLINT
- ~RE();
-
- // Returns the string representation of the regex.
- const char* pattern() const { return pattern_; }
-
- // FullMatch(str, re) returns true iff regular expression re matches
- // the entire str.
- // PartialMatch(str, re) returns true iff regular expression re
- // matches a substring of str (including str itself).
- //
- // TODO(wan@google.com): make FullMatch() and PartialMatch() work
- // when str contains NUL characters.
- static bool FullMatch(const ::std::string& str, const RE& re) {
- return FullMatch(str.c_str(), re);
- }
- static bool PartialMatch(const ::std::string& str, const RE& re) {
- return PartialMatch(str.c_str(), re);
- }
-
-#if GTEST_HAS_GLOBAL_STRING
-
- static bool FullMatch(const ::string& str, const RE& re) {
- return FullMatch(str.c_str(), re);
- }
- static bool PartialMatch(const ::string& str, const RE& re) {
- return PartialMatch(str.c_str(), re);
- }
-
-#endif // GTEST_HAS_GLOBAL_STRING
-
- static bool FullMatch(const char* str, const RE& re);
- static bool PartialMatch(const char* str, const RE& re);
-
- private:
- void Init(const char* regex);
-
- // We use a const char* instead of a string, as Google Test may be used
- // where string is not available. We also do not use Google Test's own
- // String type here, in order to simplify dependencies between the
- // files.
- const char* pattern_;
- bool is_valid_;
-
-#if GTEST_USES_POSIX_RE
-
- regex_t full_regex_; // For FullMatch().
- regex_t partial_regex_; // For PartialMatch().
-
-#else // GTEST_USES_SIMPLE_RE
-
- const char* full_pattern_; // For FullMatch();
-
-#endif
-
- GTEST_DISALLOW_ASSIGN_(RE);
-};
-
-// Formats a source file path and a line number as they would appear
-// in an error message from the compiler used to compile this code.
-GTEST_API_ ::std::string FormatFileLocation(const char* file, int line);
-
-// Formats a file location for compiler-independent XML output.
-// Although this function is not platform dependent, we put it next to
-// FormatFileLocation in order to contrast the two functions.
-GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file,
- int line);
-
-// Defines logging utilities:
-// GTEST_LOG_(severity) - logs messages at the specified severity level. The
-// message itself is streamed into the macro.
-// LogToStderr() - directs all log messages to stderr.
-// FlushInfoLog() - flushes informational log messages.
-
-enum GTestLogSeverity {
- GTEST_INFO,
- GTEST_WARNING,
- GTEST_ERROR,
- GTEST_FATAL
-};
-
-// Formats log entry severity, provides a stream object for streaming the
-// log message, and terminates the message with a newline when going out of
-// scope.
-class GTEST_API_ GTestLog {
- public:
- GTestLog(GTestLogSeverity severity, const char* file, int line);
-
- // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program.
- ~GTestLog();
-
- ::std::ostream& GetStream() { return ::std::cerr; }
-
- private:
- const GTestLogSeverity severity_;
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog);
-};
-
-#define GTEST_LOG_(severity) \
- ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \
- __FILE__, __LINE__).GetStream()
-
-inline void LogToStderr() {}
-inline void FlushInfoLog() { fflush(NULL); }
-
-// INTERNAL IMPLEMENTATION - DO NOT USE.
-//
-// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition
-// is not satisfied.
-// Synopsys:
-// GTEST_CHECK_(boolean_condition);
-// or
-// GTEST_CHECK_(boolean_condition) << "Additional message";
-//
-// This checks the condition and if the condition is not satisfied
-// it prints message about the condition violation, including the
-// condition itself, plus additional message streamed into it, if any,
-// and then it aborts the program. It aborts the program irrespective of
-// whether it is built in the debug mode or not.
-#define GTEST_CHECK_(condition) \
- GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
- if (::testing::internal::IsTrue(condition)) \
- ; \
- else \
- GTEST_LOG_(FATAL) << "Condition " #condition " failed. "
-
-// An all-mode assert to verify that the given POSIX-style function
-// call returns 0 (indicating success). Known limitation: this
-// doesn't expand to a balanced 'if' statement, so enclose the macro
-// in {} if you need to use it as the only statement in an 'if'
-// branch.
-#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \
- if (const int gtest_error = (posix_call)) \
- GTEST_LOG_(FATAL) << #posix_call << "failed with error " \
- << gtest_error
-
-// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE.
-//
-// Use ImplicitCast_ as a safe version of static_cast for upcasting in
-// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a
-// const Foo*). When you use ImplicitCast_, the compiler checks that
-// the cast is safe. Such explicit ImplicitCast_s are necessary in
-// surprisingly many situations where C++ demands an exact type match
-// instead of an argument type convertable to a target type.
-//
-// The syntax for using ImplicitCast_ is the same as for static_cast:
-//
-// ImplicitCast_<ToType>(expr)
-//
-// ImplicitCast_ would have been part of the C++ standard library,
-// but the proposal was submitted too late. It will probably make
-// its way into the language in the future.
-//
-// This relatively ugly name is intentional. It prevents clashes with
-// similar functions users may have (e.g., implicit_cast). The internal
-// namespace alone is not enough because the function can be found by ADL.
-template<typename To>
-inline To ImplicitCast_(To x) { return x; }
-
-// When you upcast (that is, cast a pointer from type Foo to type
-// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts
-// always succeed. When you downcast (that is, cast a pointer from
-// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because
-// how do you know the pointer is really of type SubclassOfFoo? It
-// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus,
-// when you downcast, you should use this macro. In debug mode, we
-// use dynamic_cast<> to double-check the downcast is legal (we die
-// if it's not). In normal mode, we do the efficient static_cast<>
-// instead. Thus, it's important to test in debug mode to make sure
-// the cast is legal!
-// This is the only place in the code we should use dynamic_cast<>.
-// In particular, you SHOULDN'T be using dynamic_cast<> in order to
-// do RTTI (eg code like this:
-// if (dynamic_cast<Subclass1>(foo)) HandleASubclass1Object(foo);
-// if (dynamic_cast<Subclass2>(foo)) HandleASubclass2Object(foo);
-// You should design the code some other way not to need this.
-//
-// This relatively ugly name is intentional. It prevents clashes with
-// similar functions users may have (e.g., down_cast). The internal
-// namespace alone is not enough because the function can be found by ADL.
-template<typename To, typename From> // use like this: DownCast_<T*>(foo);
-inline To DownCast_(From* f) { // so we only accept pointers
- // Ensures that To is a sub-type of From *. This test is here only
- // for compile-time type checking, and has no overhead in an
- // optimized build at run-time, as it will be optimized away
- // completely.
- if (false) {
- const To to = NULL;
- ::testing::internal::ImplicitCast_<From*>(to);
- }
-
-#if GTEST_HAS_RTTI
- // RTTI: debug mode only!
- GTEST_CHECK_(f == NULL || dynamic_cast<To>(f) != NULL);
-#endif
- return static_cast<To>(f);
-}
-
-// Downcasts the pointer of type Base to Derived.
-// Derived must be a subclass of Base. The parameter MUST
-// point to a class of type Derived, not any subclass of it.
-// When RTTI is available, the function performs a runtime
-// check to enforce this.
-template <class Derived, class Base>
-Derived* CheckedDowncastToActualType(Base* base) {
-#if GTEST_HAS_RTTI
- GTEST_CHECK_(typeid(*base) == typeid(Derived));
- return dynamic_cast<Derived*>(base); // NOLINT
-#else
- return static_cast<Derived*>(base); // Poor man's downcast.
-#endif
-}
-
-#if GTEST_HAS_STREAM_REDIRECTION
-
-// Defines the stderr capturer:
-// CaptureStdout - starts capturing stdout.
-// GetCapturedStdout - stops capturing stdout and returns the captured string.
-// CaptureStderr - starts capturing stderr.
-// GetCapturedStderr - stops capturing stderr and returns the captured string.
-//
-GTEST_API_ void CaptureStdout();
-GTEST_API_ String GetCapturedStdout();
-GTEST_API_ void CaptureStderr();
-GTEST_API_ String GetCapturedStderr();
-
-#endif // GTEST_HAS_STREAM_REDIRECTION
-
-
-#if GTEST_HAS_DEATH_TEST
-
-// A copy of all command line arguments. Set by InitGoogleTest().
-extern ::std::vector<String> g_argvs;
-
-// GTEST_HAS_DEATH_TEST implies we have ::std::string.
-const ::std::vector<String>& GetArgvs();
-
-#endif // GTEST_HAS_DEATH_TEST
-
-// Defines synchronization primitives.
-
-#if GTEST_HAS_PTHREAD
-
-// Sleeps for (roughly) n milli-seconds. This function is only for
-// testing Google Test's own constructs. Don't use it in user tests,
-// either directly or indirectly.
-inline void SleepMilliseconds(int n) {
- const timespec time = {
- 0, // 0 seconds.
- n * 1000L * 1000L, // And n ms.
- };
- nanosleep(&time, NULL);
-}
-
-// Allows a controller thread to pause execution of newly created
-// threads until notified. Instances of this class must be created
-// and destroyed in the controller thread.
-//
-// This class is only for testing Google Test's own constructs. Do not
-// use it in user tests, either directly or indirectly.
-class Notification {
- public:
- Notification() : notified_(false) {}
-
- // Notifies all threads created with this notification to start. Must
- // be called from the controller thread.
- void Notify() { notified_ = true; }
-
- // Blocks until the controller thread notifies. Must be called from a test
- // thread.
- void WaitForNotification() {
- while(!notified_) {
- SleepMilliseconds(10);
- }
- }
-
- private:
- volatile bool notified_;
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification);
-};
-
-// As a C-function, ThreadFuncWithCLinkage cannot be templated itself.
-// Consequently, it cannot select a correct instantiation of ThreadWithParam
-// in order to call its Run(). Introducing ThreadWithParamBase as a
-// non-templated base class for ThreadWithParam allows us to bypass this
-// problem.
-class ThreadWithParamBase {
- public:
- virtual ~ThreadWithParamBase() {}
- virtual void Run() = 0;
-};
-
-// pthread_create() accepts a pointer to a function type with the C linkage.
-// According to the Standard (7.5/1), function types with different linkages
-// are different even if they are otherwise identical. Some compilers (for
-// example, SunStudio) treat them as different types. Since class methods
-// cannot be defined with C-linkage we need to define a free C-function to
-// pass into pthread_create().
-extern "C" inline void* ThreadFuncWithCLinkage(void* thread) {
- static_cast<ThreadWithParamBase*>(thread)->Run();
- return NULL;
-}
-
-// Helper class for testing Google Test's multi-threading constructs.
-// To use it, write:
-//
-// void ThreadFunc(int param) { /* Do things with param */ }
-// Notification thread_can_start;
-// ...
-// // The thread_can_start parameter is optional; you can supply NULL.
-// ThreadWithParam<int> thread(&ThreadFunc, 5, &thread_can_start);
-// thread_can_start.Notify();
-//
-// These classes are only for testing Google Test's own constructs. Do
-// not use them in user tests, either directly or indirectly.
-template <typename T>
-class ThreadWithParam : public ThreadWithParamBase {
- public:
- typedef void (*UserThreadFunc)(T);
-
- ThreadWithParam(
- UserThreadFunc func, T param, Notification* thread_can_start)
- : func_(func),
- param_(param),
- thread_can_start_(thread_can_start),
- finished_(false) {
- ThreadWithParamBase* const base = this;
- // The thread can be created only after all fields except thread_
- // have been initialized.
- GTEST_CHECK_POSIX_SUCCESS_(
- pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base));
- }
- ~ThreadWithParam() { Join(); }
-
- void Join() {
- if (!finished_) {
- GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0));
- finished_ = true;
- }
- }
-
- virtual void Run() {
- if (thread_can_start_ != NULL)
- thread_can_start_->WaitForNotification();
- func_(param_);
- }
-
- private:
- const UserThreadFunc func_; // User-supplied thread function.
- const T param_; // User-supplied parameter to the thread function.
- // When non-NULL, used to block execution until the controller thread
- // notifies.
- Notification* const thread_can_start_;
- bool finished_; // true iff we know that the thread function has finished.
- pthread_t thread_; // The native thread object.
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam);
-};
-
-// MutexBase and Mutex implement mutex on pthreads-based platforms. They
-// are used in conjunction with class MutexLock:
-//
-// Mutex mutex;
-// ...
-// MutexLock lock(&mutex); // Acquires the mutex and releases it at the end
-// // of the current scope.
-//
-// MutexBase implements behavior for both statically and dynamically
-// allocated mutexes. Do not use MutexBase directly. Instead, write
-// the following to define a static mutex:
-//
-// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex);
-//
-// You can forward declare a static mutex like this:
-//
-// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex);
-//
-// To create a dynamic mutex, just define an object of type Mutex.
-class MutexBase {
- public:
- // Acquires this mutex.
- void Lock() {
- GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_));
- owner_ = pthread_self();
- }
-
- // Releases this mutex.
- void Unlock() {
- // We don't protect writing to owner_ here, as it's the caller's
- // responsibility to ensure that the current thread holds the
- // mutex when this is called.
- owner_ = 0;
- GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_));
- }
-
- // Does nothing if the current thread holds the mutex. Otherwise, crashes
- // with high probability.
- void AssertHeld() const {
- GTEST_CHECK_(owner_ == pthread_self())
- << "The current thread is not holding the mutex @" << this;
- }
-
- // A static mutex may be used before main() is entered. It may even
- // be used before the dynamic initialization stage. Therefore we
- // must be able to initialize a static mutex object at link time.
- // This means MutexBase has to be a POD and its member variables
- // have to be public.
- public:
- pthread_mutex_t mutex_; // The underlying pthread mutex.
- pthread_t owner_; // The thread holding the mutex; 0 means no one holds it.
-};
-
-// Forward-declares a static mutex.
-# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
- extern ::testing::internal::MutexBase mutex
-
-// Defines and statically (i.e. at link time) initializes a static mutex.
-# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \
- ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, 0 }
-
-// The Mutex class can only be used for mutexes created at runtime. It
-// shares its API with MutexBase otherwise.
-class Mutex : public MutexBase {
- public:
- Mutex() {
- GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL));
- owner_ = 0;
- }
- ~Mutex() {
- GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_));
- }
-
- private:
- GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex);
-};
-
-// We cannot name this class MutexLock as the ctor declaration would
-// conflict with a macro named MutexLock, which is defined on some
-// platforms. Hence the typedef trick below.
-class GTestMutexLock {
- public:
- explicit GTestMutexLock(MutexBase* mutex)
- : mutex_(mutex) { mutex_->Lock(); }
-
- ~GTestMutexLock() { mutex_->Unlock(); }
-
- private:
- MutexBase* const mutex_;
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock);
-};
-
-typedef GTestMutexLock MutexLock;
-
-// Helpers for ThreadLocal.
-
-// pthread_key_create() requires DeleteThreadLocalValue() to have
-// C-linkage. Therefore it cannot be templatized to access
-// ThreadLocal<T>. Hence the need for class
-// ThreadLocalValueHolderBase.
-class ThreadLocalValueHolderBase {
- public:
- virtual ~ThreadLocalValueHolderBase() {}
-};
-
-// Called by pthread to delete thread-local data stored by
-// pthread_setspecific().
-extern "C" inline void DeleteThreadLocalValue(void* value_holder) {
- delete static_cast<ThreadLocalValueHolderBase*>(value_holder);
-}
-
-// Implements thread-local storage on pthreads-based systems.
-//
-// // Thread 1
-// ThreadLocal<int> tl(100); // 100 is the default value for each thread.
-//
-// // Thread 2
-// tl.set(150); // Changes the value for thread 2 only.
-// EXPECT_EQ(150, tl.get());
-//
-// // Thread 1
-// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value.
-// tl.set(200);
-// EXPECT_EQ(200, tl.get());
-//
-// The template type argument T must have a public copy constructor.
-// In addition, the default ThreadLocal constructor requires T to have
-// a public default constructor.
-//
-// An object managed for a thread by a ThreadLocal instance is deleted
-// when the thread exits. Or, if the ThreadLocal instance dies in
-// that thread, when the ThreadLocal dies. It's the user's
-// responsibility to ensure that all other threads using a ThreadLocal
-// have exited when it dies, or the per-thread objects for those
-// threads will not be deleted.
-//
-// Google Test only uses global ThreadLocal objects. That means they
-// will die after main() has returned. Therefore, no per-thread
-// object managed by Google Test will be leaked as long as all threads
-// using Google Test have exited when main() returns.
-template <typename T>
-class ThreadLocal {
- public:
- ThreadLocal() : key_(CreateKey()),
- default_() {}
- explicit ThreadLocal(const T& value) : key_(CreateKey()),
- default_(value) {}
-
- ~ThreadLocal() {
- // Destroys the managed object for the current thread, if any.
- DeleteThreadLocalValue(pthread_getspecific(key_));
-
- // Releases resources associated with the key. This will *not*
- // delete managed objects for other threads.
- GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_));
- }
-
- T* pointer() { return GetOrCreateValue(); }
- const T* pointer() const { return GetOrCreateValue(); }
- const T& get() const { return *pointer(); }
- void set(const T& value) { *pointer() = value; }
-
- private:
- // Holds a value of type T.
- class ValueHolder : public ThreadLocalValueHolderBase {
- public:
- explicit ValueHolder(const T& value) : value_(value) {}
-
- T* pointer() { return &value_; }
-
- private:
- T value_;
- GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder);
- };
-
- static pthread_key_t CreateKey() {
- pthread_key_t key;
- // When a thread exits, DeleteThreadLocalValue() will be called on
- // the object managed for that thread.
- GTEST_CHECK_POSIX_SUCCESS_(
- pthread_key_create(&key, &DeleteThreadLocalValue));
- return key;
- }
-
- T* GetOrCreateValue() const {
- ThreadLocalValueHolderBase* const holder =
- static_cast<ThreadLocalValueHolderBase*>(pthread_getspecific(key_));
- if (holder != NULL) {
- return CheckedDowncastToActualType<ValueHolder>(holder)->pointer();
- }
-
- ValueHolder* const new_holder = new ValueHolder(default_);
- ThreadLocalValueHolderBase* const holder_base = new_holder;
- GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base));
- return new_holder->pointer();
- }
-
- // A key pthreads uses for looking up per-thread values.
- const pthread_key_t key_;
- const T default_; // The default value for each thread.
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal);
-};
-
-# define GTEST_IS_THREADSAFE 1
-
-#else // GTEST_HAS_PTHREAD
-
-// A dummy implementation of synchronization primitives (mutex, lock,
-// and thread-local variable). Necessary for compiling Google Test where
-// mutex is not supported - using Google Test in multiple threads is not
-// supported on such platforms.
-
-class Mutex {
- public:
- Mutex() {}
- void AssertHeld() const {}
-};
-
-# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
- extern ::testing::internal::Mutex mutex
-
-# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex
-
-class GTestMutexLock {
- public:
- explicit GTestMutexLock(Mutex*) {} // NOLINT
-};
-
-typedef GTestMutexLock MutexLock;
-
-template <typename T>
-class ThreadLocal {
- public:
- ThreadLocal() : value_() {}
- explicit ThreadLocal(const T& value) : value_(value) {}
- T* pointer() { return &value_; }
- const T* pointer() const { return &value_; }
- const T& get() const { return value_; }
- void set(const T& value) { value_ = value; }
- private:
- T value_;
-};
-
-// The above synchronization primitives have dummy implementations.
-// Therefore Google Test is not thread-safe.
-# define GTEST_IS_THREADSAFE 0
-
-#endif // GTEST_HAS_PTHREAD
-
-// Returns the number of threads running in the process, or 0 to indicate that
-// we cannot detect it.
-GTEST_API_ size_t GetThreadCount();
-
-// Passing non-POD classes through ellipsis (...) crashes the ARM
-// compiler and generates a warning in Sun Studio. The Nokia Symbian
-// and the IBM XL C/C++ compiler try to instantiate a copy constructor
-// for objects passed through ellipsis (...), failing for uncopyable
-// objects. We define this to ensure that only POD is passed through
-// ellipsis on these systems.
-#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || defined(__SUNPRO_CC)
-// We lose support for NULL detection where the compiler doesn't like
-// passing non-POD classes through ellipsis (...).
-# define GTEST_ELLIPSIS_NEEDS_POD_ 1
-#else
-# define GTEST_CAN_COMPARE_NULL 1
-#endif
-
-// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between
-// const T& and const T* in a function template. These compilers
-// _can_ decide between class template specializations for T and T*,
-// so a tr1::type_traits-like is_pointer works.
-#if defined(__SYMBIAN32__) || defined(__IBMCPP__)
-# define GTEST_NEEDS_IS_POINTER_ 1
-#endif
-
-template <bool bool_value>
-struct bool_constant {
- typedef bool_constant<bool_value> type;
- static const bool value = bool_value;
-};
-template <bool bool_value> const bool bool_constant<bool_value>::value;
-
-typedef bool_constant<false> false_type;
-typedef bool_constant<true> true_type;
-
-template <typename T>
-struct is_pointer : public false_type {};
-
-template <typename T>
-struct is_pointer<T*> : public true_type {};
-
-template <typename Iterator>
-struct IteratorTraits {
- typedef typename Iterator::value_type value_type;
-};
-
-template <typename T>
-struct IteratorTraits<T*> {
- typedef T value_type;
-};
-
-template <typename T>
-struct IteratorTraits<const T*> {
- typedef T value_type;
-};
-
-#if GTEST_OS_WINDOWS
-# define GTEST_PATH_SEP_ "\\"
-# define GTEST_HAS_ALT_PATH_SEP_ 1
-// The biggest signed integer type the compiler supports.
-typedef __int64 BiggestInt;
-#else
-# define GTEST_PATH_SEP_ "/"
-# define GTEST_HAS_ALT_PATH_SEP_ 0
-typedef long long BiggestInt; // NOLINT
-#endif // GTEST_OS_WINDOWS
-
-// Utilities for char.
-
-// isspace(int ch) and friends accept an unsigned char or EOF. char
-// may be signed, depending on the compiler (or compiler flags).
-// Therefore we need to cast a char to unsigned char before calling
-// isspace(), etc.
-
-inline bool IsAlpha(char ch) {
- return isalpha(static_cast<unsigned char>(ch)) != 0;
-}
-inline bool IsAlNum(char ch) {
- return isalnum(static_cast<unsigned char>(ch)) != 0;
-}
-inline bool IsDigit(char ch) {
- retur
<TRUNCATED>
[18/50] [abbrv] git commit: MAPREDUCE-5978. native-task: remove test
case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
Posted by ar...@apache.org.
MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616116 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d4fb172
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d4fb172
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d4fb172
Branch: refs/heads/HDFS-6581
Commit: 4d4fb1723fd825df4ad2488e4ecde0c69359e83a
Parents: 432f641
Author: Sean Zhong <se...@apache.org>
Authored: Wed Aug 6 07:40:24 2014 +0000
Committer: Sean Zhong <se...@apache.org>
Committed: Wed Aug 6 07:40:24 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 3 +-
.../nativetask/compresstest/CompressTest.java | 33 -----------------
.../test/resources/test-bzip2-compress-conf.xml | 39 --------------------
.../resources/test-default-compress-conf.xml | 39 --------------------
4 files changed, 2 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 4b77262..462ac8b 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -10,4 +10,5 @@ MAPREDUCE-5991. native-task should not run unit tests if native profile is not e
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
-MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
\ No newline at end of file
+MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
+MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index 0406375..b98e2de 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -68,23 +68,6 @@ public class CompressTest {
}
@Test
- public void testBzip2Compress() throws Exception {
- final Configuration nativeconf = ScenarioConfiguration.getNativeConfiguration();
- nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job nativejob = CompressMapper.getCompressJob("nativebzip2", nativeconf);
- nativejob.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
- CompressMapper.outputFileDir + "hadoopbzip2");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
public void testLz4Compress() throws Exception {
final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
@@ -100,22 +83,6 @@ public class CompressTest {
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
- @Test
- public void testDefaultCompress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativedefault", nativeConf);
- nativeJob.waitForCompletion(true);
-
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadoopdefault", hadoopConf);
- hadoopJob.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativedefault",
- CompressMapper.outputFileDir + "hadoopdefault");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
@Before
public void startUp() throws Exception {
final ScenarioConfiguration conf = new ScenarioConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml
deleted file mode 100644
index 2484984..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.BZip2Codec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml
deleted file mode 100644
index 765daf1..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.DefaultCodec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>
[21/50] [abbrv] git commit: MAPREDUCE-6035. native-task:
sources/test-sources jar distribution. Contributed by Manu Zhang.
Posted by ar...@apache.org.
MAPREDUCE-6035. native-task: sources/test-sources jar distribution. Contributed by Manu Zhang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617879 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eace4dd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eace4dd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eace4dd5
Branch: refs/heads/HDFS-6581
Commit: eace4dd50b1214a8842e47c522da258e1746386a
Parents: 808bf8b
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Aug 14 04:56:43 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Aug 14 04:56:43 2014 +0000
----------------------------------------------------------------------
.../src/main/resources/assemblies/hadoop-mapreduce-dist.xml | 7 +++++++
hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 +
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eace4dd5/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index 0cb01ea..749e16a 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -136,6 +136,13 @@
</includes>
</fileSet>
<fileSet>
+ <directory>hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+ <includes>
+ <include>*-sources.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
<directory>hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eace4dd5/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 123eb6d..95f7858 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -14,3 +14,4 @@ MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu
MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd)
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
+MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd)
[10/50] [abbrv] git commit: MAPREDUCE-5997. native-task: Use
DirectBufferPool from Hadoop Common. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613034 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77acc70d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77acc70d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77acc70d
Branch: refs/heads/HDFS-6581
Commit: 77acc70df5d64055ea809222f3d2f0d66c611196
Parents: 5149a8a
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jul 24 08:20:25 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jul 24 08:20:25 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../mapred/nativetask/NativeBatchProcessor.java | 8 +-
.../nativetask/buffer/DirectBufferPool.java | 93 ---------
.../mapred/nativetask/buffer/InputBuffer.java | 17 +-
.../nativetask/buffer/TestDirectBufferPool.java | 201 -------------------
5 files changed, 20 insertions(+), 300 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index cea5a76..e12f743 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -4,3 +4,4 @@ Changes for Hadoop Native Map Output Collector
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd)
+MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index fd68ea6..837da0e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -24,12 +24,13 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
-import org.apache.hadoop.mapred.nativetask.buffer.DirectBufferPool;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
+import org.apache.hadoop.util.DirectBufferPool;
/**
* used to create channel, transfer data and command between Java and native
@@ -126,9 +127,8 @@ public class NativeBatchProcessor implements INativeHandler {
NativeRuntime.releaseNativeObject(nativeHandlerAddr);
nativeHandlerAddr = 0;
}
- if (null != in && null != in.getByteBuffer() && in.getByteBuffer().isDirect()) {
- DirectBufferPool.getInstance().returnBuffer(in.getByteBuffer());
- }
+ IOUtils.cleanup(LOG, in);
+ in = null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
deleted file mode 100644
index bd3c6bb..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.buffer;
-
-import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.nio.ByteBuffer;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-
-/**
- * as direct buffer memory is not collected by GC, we keep a pool
- * to reuse direct buffers
- */
-public class DirectBufferPool {
-
- private static DirectBufferPool directBufferPool = null;
- private static Log LOG = LogFactory.getLog(DirectBufferPool.class);
- private ConcurrentMap<Integer, Queue<WeakReference<ByteBuffer>>> bufferMap = new ConcurrentHashMap<Integer, Queue<WeakReference<ByteBuffer>>>();
-
- private DirectBufferPool() {
- }
-
- public static synchronized DirectBufferPool getInstance() {
- if (null == directBufferPool) {
- directBufferPool = new DirectBufferPool();
- }
- return directBufferPool;
- }
-
- public static void destoryInstance(){
- directBufferPool = null;
- }
-
- public synchronized ByteBuffer borrowBuffer(int capacity) throws IOException {
- Queue<WeakReference<ByteBuffer>> list = bufferMap.get(capacity);
- if (null == list) {
- return ByteBuffer.allocateDirect(capacity);
- }
- WeakReference<ByteBuffer> ref;
- while ((ref = list.poll()) != null) {
- ByteBuffer buf = ref.get();
- if (buf != null) {
- return buf;
- }
- }
- return ByteBuffer.allocateDirect(capacity);
- }
-
- public void returnBuffer(ByteBuffer buffer) throws IOException {
- if (null == buffer || !buffer.isDirect()) {
- throw new IOException("the buffer is null or the buffer returned is not direct buffer");
- }
-
- buffer.clear();
- int capacity = buffer.capacity();
- Queue<WeakReference<ByteBuffer>> list = bufferMap.get(capacity);
- if (null == list) {
- list = new ConcurrentLinkedQueue<WeakReference<ByteBuffer>>();
- Queue<WeakReference<ByteBuffer>> prev = bufferMap.putIfAbsent(capacity, list);
- if (prev != null) {
- list = prev;
- }
- }
- list.add(new WeakReference<ByteBuffer>(buffer));
- }
-
- int getBufCountsForCapacity(int capacity) {
- return bufferMap.get(capacity).size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
index eb15164..82193b8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java
@@ -18,11 +18,16 @@
package org.apache.hadoop.mapred.nativetask.buffer;
+import org.apache.hadoop.util.DirectBufferPool;
+
+import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-public class InputBuffer {
+public class InputBuffer implements Closeable {
+
+ static DirectBufferPool bufferPool = new DirectBufferPool();
private ByteBuffer byteBuffer;
private final BufferType type;
@@ -36,7 +41,7 @@ public class InputBuffer {
switch (type) {
case DIRECT_BUFFER:
- this.byteBuffer = DirectBufferPool.getInstance().borrowBuffer(capacity);
+ this.byteBuffer = bufferPool.getBuffer(capacity);
this.byteBuffer.order(ByteOrder.BIG_ENDIAN);
break;
case HEAP_BUFFER:
@@ -118,4 +123,12 @@ public class InputBuffer {
}
return byteBuffer.array();
}
+
+ @Override
+ public void close() {
+ if (byteBuffer != null && byteBuffer.isDirect()) {
+ bufferPool.returnBuffer(byteBuffer);
+ byteBuffer = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestDirectBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestDirectBufferPool.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestDirectBufferPool.java
deleted file mode 100644
index 09c1ef5..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestDirectBufferPool.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred.nativetask.buffer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.ArrayList;
-
-import org.junit.Test;
-
-public class TestDirectBufferPool {
-
- @Test
- public void testGetInstance() throws Exception {
- final int num = 100;
- List<DirectBufferPool> pools = new ArrayList<DirectBufferPool>();
- Thread[] list = new Thread[num];
- for (int i = 0; i < num; i++) {
- Thread t = getPoolThread(pools);
- t.start();
- list[i] = t;
- }
- for (int i = 0; i < num; i++) {
- try {
- list[i].join(10000);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- DirectBufferPool p1 = pools.get(0);
- assertNotNull(p1);
- for (int i = 1; i < pools.size(); i++) {
- DirectBufferPool p2 = pools.get(i);
- assertNotNull(p2);
- assertSame(p1, p2);
- }
- }
-
- private Thread getPoolThread(final List<DirectBufferPool> pools) {
- Thread t = new Thread() {
- public void run() {
- pools.add(DirectBufferPool.getInstance());
- }
- };
- return t;
- }
-
-
- @Test
- public void testBufBorrow() throws IOException {
- final DirectBufferPool bufferPool = DirectBufferPool.getInstance();
- ByteBuffer b1 = bufferPool.borrowBuffer(100);
- assertTrue(b1.isDirect());
- assertEquals(0, b1.position());
- assertEquals(100, b1.capacity());
- bufferPool.returnBuffer(b1);
- ByteBuffer b2 = bufferPool.borrowBuffer(100);
- assertTrue(b2.isDirect());
- assertEquals(0, b2.position());
- assertEquals(100, b2.capacity());
- assertSame(b1, b2);
-
- ByteBuffer b3 = bufferPool.borrowBuffer(100);
- assertTrue(b3.isDirect());
- assertEquals(0, b3.position());
- assertEquals(100, b3.capacity());
- assertNotSame(b2, b3);
- bufferPool.returnBuffer(b2);
- bufferPool.returnBuffer(b3);
- }
-
- @Test
- public void testBufReset() throws IOException {
- final DirectBufferPool bufferPool = DirectBufferPool.getInstance();
- ByteBuffer b1 = bufferPool.borrowBuffer(100);
- assertTrue(b1.isDirect());
- assertEquals(0, b1.position());
- assertEquals(100, b1.capacity());
- b1.putInt(1);
- assertEquals(4, b1.position());
- bufferPool.returnBuffer(b1);
- ByteBuffer b2 = bufferPool.borrowBuffer(100);
- assertSame(b1, b2);
- assertTrue(b2.isDirect());
- assertEquals(0, b2.position());
- assertEquals(100, b2.capacity());
- }
-
- @Test
- public void testBufReturn() throws IOException {
- final DirectBufferPool bufferPool = DirectBufferPool.getInstance();
- int numOfBufs = 100;
- int capacity = 100;
- final ByteBuffer[] bufs = new ByteBuffer[numOfBufs];
- for (int i = 0; i < numOfBufs; i++) {
- bufs[i] = bufferPool.borrowBuffer(capacity);
- }
-
- assertEquals(0, bufferPool.getBufCountsForCapacity(capacity));
-
-
- int numOfThreads = numOfBufs;
- Thread[] list = new Thread[numOfThreads];
- for (int i = 0; i < numOfThreads; i++) {
- Thread t = retBufThread(bufferPool, bufs, i);
- t.start();
- list[i] = t;
- }
- for (int i = 0; i < numOfThreads; i++) {
- try {
- list[i].join(10000);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- assertEquals(numOfBufs, bufferPool.getBufCountsForCapacity(capacity));
- }
-
- private Thread retBufThread(final DirectBufferPool bufferPool, final ByteBuffer[] bufs, final int i) {
- Thread t = new Thread(new Runnable(){
- @Override
- public void run() {
- try {
- bufferPool.returnBuffer(bufs[i]);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- return t;
- }
-
- @Test
- public void testBufException() {
- final DirectBufferPool bufferPool = DirectBufferPool.getInstance();
- boolean thrown = false;
- try {
- bufferPool.returnBuffer(null);
- } catch (IOException e) {
- thrown = true;
- }
- assertEquals(true, thrown);
-
- thrown = false;
- ByteBuffer buf = ByteBuffer.allocate(100);
- try {
- bufferPool.returnBuffer(buf);
- } catch (IOException e) {
- thrown = true;
- }
- assertEquals(true, thrown);
- }
-
- @Test
- public void testBufWeakRefClear() throws IOException {
- final DirectBufferPool bufferPool = DirectBufferPool.getInstance();
- int numOfBufs = 100;
- int capacity = 100;
- ByteBuffer[] list = new ByteBuffer[capacity];
- for (int i = 0; i < numOfBufs; i++) {
- list[i] = bufferPool.borrowBuffer(capacity);
- }
- for (int i = 0; i < numOfBufs; i++) {
- bufferPool.returnBuffer(list[i]);
- list[i] = null;
- }
-
- assertEquals(numOfBufs, bufferPool.getBufCountsForCapacity(capacity));
-
- for (int i = 0; i < 3; i++) {
- System.gc();
- }
-
- ByteBuffer b = bufferPool.borrowBuffer(capacity);
- assertEquals(0, bufferPool.getBufCountsForCapacity(capacity));
- }
-}
[12/50] [abbrv] git commit: MAPREDUCE-5991. native-task should not
run unit tests if native profile is not enabled. Contributed by Binglin
Chang.
Posted by ar...@apache.org.
MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. Contributed by Binglin Chang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613072 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea9e5b7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea9e5b7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea9e5b7b
Branch: refs/heads/HDFS-6581
Commit: ea9e5b7baae34c7ba660681d1e5aceb9c1047e6f
Parents: 6635180
Author: Binglin Chang <bc...@apache.org>
Authored: Thu Jul 24 11:44:09 2014 +0000
Committer: Binglin Chang <bc...@apache.org>
Committed: Thu Jul 24 11:44:09 2014 +0000
----------------------------------------------------------------------
.../hadoop-mapreduce-client-common/pom.xml | 11 ++++++
.../hadoop-mapreduce-client-nativetask/pom.xml | 35 ++++++++++++++++----
hadoop-project/pom.xml | 6 ++++
3 files changed, 45 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 7ef7d3b..48db5c8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -84,6 +84,17 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 40b6520..2cb483e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -63,6 +63,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -82,14 +88,7 @@
<include>**/buffer/Test*.java</include>
<include>**/handlers/Test*.java</include>
<include>**/serde/Test*.java</include>
- <include>**/combinertest/*Test.java</include>
- <include>**/compresstest/*Test.java</include>
- <include>**/nonsorttest/*Test.java</include>
- <include>**/kvtest/*Test.java</include>
</includes>
- <additionalClasspathElements>
- <additionalClasspathElement>${basedir}/../hadoop-mapreduce-client-common/target/classes</additionalClasspathElement>
- </additionalClasspathElements>
</configuration>
</plugin>
</plugins>
@@ -184,6 +183,28 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ <includes>
+ <include>**/TestTaskContext.java</include>
+ <include>**/buffer/Test*.java</include>
+ <include>**/handlers/Test*.java</include>
+ <include>**/serde/Test*.java</include>
+ <include>**/combinertest/*Test.java</include>
+ <include>**/compresstest/*Test.java</include>
+ <include>**/nonsorttest/*Test.java</include>
+ <include>**/kvtest/*Test.java</include>
+ </includes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2dc83dd..947be86 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -136,6 +136,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${project.version}</version>
</dependency>
[07/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into
standard directory layout. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
new file mode 100644
index 0000000..b27f00f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
+
+public class MockValueClass implements Writable {
+ private final static int DEFAULT_ARRAY_LENGTH = 16;
+ private int a = 0;
+ private byte[] array;
+ private final LongWritable longWritable;
+ private final Text txt;
+ private final Random rand = new Random();
+
+ public MockValueClass() {
+ a = rand.nextInt();
+ array = new byte[DEFAULT_ARRAY_LENGTH];
+ rand.nextBytes(array);
+ longWritable = new LongWritable(rand.nextLong());
+ txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ public MockValueClass(byte[] seed) {
+ a = seed.length;
+ array = new byte[seed.length];
+ System.arraycopy(seed, 0, array, 0, seed.length);
+ longWritable = new LongWritable(a);
+ txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(a);
+ out.writeInt(array.length);
+ out.write(array);
+ longWritable.write(out);
+ txt.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ a = in.readInt();
+ final int length = in.readInt();
+ array = new byte[length];
+ in.readFully(array);
+ longWritable.readFields(in);
+ txt.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
new file mode 100644
index 0000000..b665971
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.IOException;
+import java.util.zip.CRC32;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+
+public class ResultVerifier {
+ /**
+ * verify the result
+ *
+ * @param sample
+ * :nativetask output
+ * @param source
+ * :yuanwenjian
+ * @throws Exception
+ */
+ public static boolean verify(String sample, String source) throws Exception {
+ FSDataInputStream sourcein = null;
+ FSDataInputStream samplein = null;
+
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+ final Path hdfssource = new Path(source);
+ final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource));
+
+ final Path hdfssample = new Path(sample);
+ final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample));
+ if (sourcepaths == null) {
+ throw new Exception("source file can not be found");
+ }
+ if (samplepaths == null) {
+ throw new Exception("sample file can not be found");
+ }
+ if (sourcepaths.length != samplepaths.length) {
+ return false;
+ }
+ for (int i = 0; i < sourcepaths.length; i++) {
+ final Path sourcepath = sourcepaths[i];
+ // op result file start with "part-r" like part-r-00000
+
+ if (!sourcepath.getName().startsWith("part-r")) {
+ continue;
+ }
+ Path samplepath = null;
+ for (int j = 0; j < samplepaths.length; j++) {
+ if (samplepaths[i].getName().equals(sourcepath.getName())) {
+ samplepath = samplepaths[i];
+ break;
+ }
+ }
+ if (samplepath == null) {
+ throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName()
+ + " , as sourcepaths has such file");
+ }
+
+ // compare
+ try {
+ if (fs.exists(sourcepath) && fs.exists(samplepath)) {
+ sourcein = fs.open(sourcepath);
+ samplein = fs.open(samplepath);
+ } else {
+ System.err.println("result file not found:" + sourcepath + " or " + samplepath);
+ return false;
+ }
+
+ CRC32 sourcecrc, samplecrc;
+ samplecrc = new CRC32();
+ sourcecrc = new CRC32();
+ final byte[] bufin = new byte[1 << 16];
+ int readnum = 0;
+ int totalRead = 0;
+ while (samplein.available() > 0) {
+ readnum = samplein.read(bufin);
+ totalRead += readnum;
+ samplecrc.update(bufin, 0, readnum);
+ }
+
+ if (0 == totalRead) {
+ throw new Exception("source " + sample + " is empty file");
+ }
+
+ totalRead = 0;
+ while (sourcein.available() > 0) {
+ readnum = sourcein.read(bufin);
+ totalRead += readnum;
+ sourcecrc.update(bufin, 0, readnum);
+ }
+ if (0 == totalRead) {
+ throw new Exception("source " + sample + " is empty file");
+ }
+
+ if (samplecrc.getValue() == sourcecrc.getValue()) {
+ ;
+ } else {
+ return false;
+ }
+ } catch (final IOException e) {
+ throw new Exception("verify exception :", e);
+ } finally {
+
+ try {
+ if (samplein != null) {
+ samplein.close();
+ }
+ if (sourcein != null) {
+ sourcein.close();
+ }
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ }
+ return true;
+ }
+
+ public static void main(String[] args) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
new file mode 100644
index 0000000..9cc473d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class ScenarioConfiguration extends Configuration {
+ public ScenarioConfiguration() {
+ super();
+ this.addResource(TestConstants.COMMON_CONF_PATH);
+ }
+
+ public void addcombinerConf() {
+ this.addResource(TestConstants.COMBINER_CONF_PATH);
+ }
+
+ public void addKVTestConf() {
+ this.addResource(TestConstants.KVTEST_CONF_PATH);
+ }
+
+ public void addNonSortTestConf() {
+ this.addResource(TestConstants.NONSORT_TEST_CONF);
+ }
+
+ public void addNativeConf() {
+ this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR,
+ TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS);
+ }
+
+ public static Configuration getNormalConfiguration() {
+ Configuration normalConf = new Configuration();
+ normalConf.addResource("common_conf.xml");
+ normalConf.addResource("normal_conf.xml");
+ return normalConf;
+ }
+
+ public static Configuration getNativeConfiguration() {
+ Configuration nativeConf = new Configuration();
+ nativeConf.addResource("common_conf.xml");
+ nativeConf.addResource("native_conf.xml");
+ return nativeConf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
new file mode 100644
index 0000000..3beb082
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+public class TestConstants {
+ // conf path
+ public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
+ public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
+ public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
+
+ public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
+ public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
+
+ public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
+ public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
+ public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
+ public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
+ public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
+ public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
+ public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
+
+ public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
+ public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
+ public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
+ public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
+ public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
+ public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
+
+ public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
+ public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
+ public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
+ public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
+ public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
+ public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
+ public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
+ public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
+ public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
+ public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
+
+ public static final String COMMON_CONF_PATH = "common_conf.xml";
+
+ public static final String FILESIZE_KEY = "kvtest.file.size";
+ public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
+
+ public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
+ public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
+ public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
+ public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
+
+}
[20/50] [abbrv] git commit: MAPREDUCE-6026. native-task: fix logging.
Contributed by Manu Zhang.
Posted by ar...@apache.org.
MAPREDUCE-6026. native-task: fix logging. Contributed by Manu Zhang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617878 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/808bf8ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/808bf8ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/808bf8ba
Branch: refs/heads/HDFS-6581
Commit: 808bf8bac1d4bdf83fb3bdf6e096b4038c9567ce
Parents: 886338f
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Aug 14 04:53:36 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Aug 14 04:53:36 2014 +0000
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/HadoopPlatform.java | 5 +++--
.../hadoop/mapred/nativetask/Platforms.java | 5 +++--
.../combinertest/LargeKVCombinerTest.java | 5 ++++-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 8 ++++++--
.../mapred/nativetask/kvtest/LargeKVTest.java | 7 +++++--
.../mapred/nativetask/kvtest/TestInputFile.java | 9 ++++++---
.../EnforceNativeOutputCollectorDelegator.java | 6 ++++--
.../src/test/resources/log4j.properties | 19 +++++++++++++++++++
9 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index c3c6ab9..123eb6d 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -13,3 +13,4 @@ MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd)
+MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
index 1c4ede5..7599bb8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -33,10 +35,9 @@ import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.*;
-import org.apache.log4j.Logger;
public class HadoopPlatform extends Platform {
- private static final Logger LOG = Logger.getLogger(HadoopPlatform.class);
+ private static final Log LOG = LogFactory.getLog(HadoopPlatform.class);
public HadoopPlatform() throws IOException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
index 154bbc8..d0a8496 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
import java.util.ServiceLoader;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
-import org.apache.log4j.Logger;
/**
@@ -34,7 +35,7 @@ import org.apache.log4j.Logger;
*/
public class Platforms {
- private static final Logger LOG = Logger.getLogger(Platforms.class);
+ private static final Log LOG = LogFactory.getLog(Platforms.class);
private static final ServiceLoader<Platform> platforms = ServiceLoader.load(Platform.class);
public static void init(Configuration conf) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index 50953e0..ff444db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativetask.combinertest;
import static org.junit.Assert.assertEquals;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,6 +36,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.junit.Test;
public class LargeKVCombinerTest {
+ private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class);
@Test
public void testLargeValueCombiner(){
@@ -57,7 +60,7 @@ public class LargeKVCombinerTest {
int max = i;
int min = Math.max(i / 4, max - 10);
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max);
+ LOG.info("===KV Size Test: min size: " + min + ", max size: " + max);
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 1e08854..7c4f3af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -37,6 +39,8 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class KVTest {
+ private static final Log LOG = LogFactory.getLog(KVTest.class);
+
private static Class<?>[] keyclasses = null;
private static Class<?>[] valueclasses = null;
private static String[] keyclassNames = null;
@@ -53,7 +57,7 @@ public class KVTest {
public static Iterable<Class<?>[]> data() {
final String valueclassesStr = nativekvtestconf
.get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
- System.out.println(valueclassesStr);
+ LOG.info(valueclassesStr);
valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
// " "
final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
@@ -69,7 +73,7 @@ public class KVTest {
}
valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
- System.out.println(keyclassesStr);
+ LOG.info(keyclassesStr);
keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
// " "
final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
index 900b058..1e120ab 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,6 +34,7 @@ import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.junit.Test;
public class LargeKVTest {
+ private static final Log LOG = LogFactory.getLog(LargeKVTest.class);
@Test
public void testKeySize() {
@@ -69,8 +72,8 @@ public class LargeKVTest {
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
- + keyClass.getName() + ", valueClass: " + valueClass.getName());
+ LOG.info("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
+ + keyClass.getName() + ", valueClass: " + valueClass.getName());
final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
valueClass, nativeConf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index a194697..80e1055 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -40,7 +42,8 @@ import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
public class TestInputFile {
-
+ private static Log LOG = LogFactory.getLog(TestInputFile.class);
+
public static class KVSizeScope {
private static final int DefaultMinNum = 1;
private static final int DefaultMaxNum = 64;
@@ -120,8 +123,8 @@ public class TestInputFile {
}
public void createSequenceTestFile(String filepath, int base, byte start) throws Exception {
- System.out.println("create file " + filepath);
- System.out.println(keyClsName + " " + valueClsName);
+ LOG.info("create file " + filepath);
+ LOG.info(keyClsName + " " + valueClsName);
Class<?> tmpkeycls, tmpvaluecls;
try {
tmpkeycls = Class.forName(keyClsName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
index a0f7d64..18ca629 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
@@ -19,9 +19,12 @@ package org.apache.hadoop.mapred.nativetask.testutil;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
+ private static final Log LOG = LogFactory.getLog(EnforceNativeOutputCollectorDelegator.class);
private boolean nativetaskloaded = false;
@Override
@@ -32,8 +35,7 @@ public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutput
nativetaskloaded = true;
} catch (final Exception e) {
nativetaskloaded = false;
- System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
- e.printStackTrace();
+ LOG.error("load nativetask lib failed, Native-Task Delegation is disabled", e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
[29/50] [abbrv] MAPREDUCE-6056. native-task: move system test working
dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml
deleted file mode 100644
index 46b3ea7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.SnappyCodec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>
[06/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into
standard directory layout. Contributed by Todd Lipcon.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
deleted file mode 100644
index 168a65d..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
+++ /dev/null
@@ -1,2 +0,0 @@
-��*�PKM��&:\xBD\xC8\xCA\xCB\xAA\xB4c`\xD5|A1+\xCB\xDC\xAA\x7F\x1D\xA4\x1Bx\x1B\xD1k\xE1\x01\x0E\xEA\xC82\xB7\xF8 \xBD.z\xEFH\xFB*\xE8\xBD\xD4\x15o\x82\xA1\x0F}\x03\x00\x17\xC7\xF7Ce*T\x1D\xC4tk\xE7\x0CS\xBF["\xADW{\xB9\xFFLM\xB2\x87\x03\xFC\xC2\xE2\x9C%\xCB\xFF\x9A\x97]\x15_N\xA3\x8F&Db\xA6\x172\x99\xF8+\xA4\xA5\xAC4n\x0D}\xA0/[\xBD\xFD\xC4\xA7\xBCj\xF8\x85\xC6V\x87B\xBA\x85\x8A\xCDZ\xFBU\x84\xD7\xBC\xAB\x84H}_v\xC4\xB2\x11\x19\xDB\x06\x93\xB5\xBE\x92E\xCE\xFA\x02RG\xE4\xA3HcI4\xAB.X\xA5\xB8a\x06\x1E.\xADS\xFB\x8A&W\xDA\xCA2^\\x90\x87\x95l]r\x99\x88b\x1EU\xC8\xE0l\x1F#h\x08\xA8\x80I\x9B\xC0E\x19%\xDE\xE5\xA6?\xC2\x83\xA0\xD6O1\xFB_\x94\xF8\x8F\xD6\xD9\x16\xE2\x91\x1E\x8EL\x1F\xA2Y\x05\x9CoMN\x0C\x86\xE2\xE9\x9A\xB16o\xF6\xF3Q\xA2\xD8\xB8\xD0\xDB\x86\x93\xAFesc\xA0h\x12\xB9E\x1BU\x12\x02OXS\x8BXn\x0EQ\xABJ\xCE\xB2k\xB1\xD7Z'\xE4\xB7[`~4\x11\xB4\xCD`\xF5%\x1F\xF9\xEEW\x88\xC5\x14<+O\x1B9B>\x11\x95<\xDE\xD5\xA4C<o\x8E\xDB\xEAz\x9FGi\xAD&\xFE\xF8\xB63\xED\x04\xA1\xE2v\x86G\
xAE\x09\x91\xE4\x03JS=\\xD1\x81}\xEE\xA3<?\x87\xDB\xC1\x8B\xFF)\xE8L\xC3\xC7\x8B\x05\x1B'\xCC)\xD4\x0D\x90{\xB9\xB7C6\xA9\x16\xE2\xF9D%\xE9!Bt]N\xFB&\xF3\xCE\x7F\xB7d8\xBCN\x191,Nm\xABb\x92\x00\xEA\xBC\xD0\xD5!\x956\xF9\x9D\x98\x86\x81QDnL\xB1\xEFxX\xA4\x1FZ\xE9uf\x7F\xF7\x8F\xCD\xC5P\x81\x17\x17\xB0\xCB\xAF\x9C\x05\x8E\xC6\xDB\x09\x1Ak\xA6\xF822\xF4\xA4t\xA1;2\xCC\xEA\xFD?0k]\xF8\xE4\x13\xD2'Z\x9F~5\x9C\xFC+k\x07\xE4k\xB9\xEDx\x9B\x17\xA2"\xA8\x85&R\x90\x1CV\xD3T\xB7b[\x81)"8\xEA\xE6\x12\xE0\x88~\xB8\x87\xA9X\xB0\x88\x19\xA5,\x88cm@\xEE\xF9.\x8A?\xF8!\xD8oR\xAB\x05\x93h3\x13\x0A\x98_E\x11\x81\xD6\xB8;P\xD8u<\x9DTv>]\\xF1\x0C\xD5\xF4\x0E#\x87}\xE3\x89\xA2\xC2\xEA\x86\x9D\xE7\xAF\xA1\xC3;\xD2\xFF\xA6\xB2!\xAB\\x90i|n\xDE\xBB>:\xC6\x08\x1D,Q\xC1;\x15\x9D>UV\x8F\xD3;\xFA\x12\xD0U\xA9\xD7<\xC6\xFDX\x8F\x10vX\xF1\xF9H7\x12Z\x1FoB\x8B\xD6\xF0\x04\xEB\xA2\xC1\xACH\xFE\xC9_R\xF5\x1D\xA8\x82\xBD\xC6\x11\x19#\x9A\xA8\x09\x95\x06\x98\xA9f\xD9\x12\xA0i\xFA\xD3\xB2\x01\xE9\x06\x14"~."z\xDF\xF2YV
#z\xEB-\xFA$[U\xA6\xE2.\xD6\xD0kf\xA3\x83\xE0\x86\xD9\xC1\x7FO\xE9\xF4\xEF\x81\x06\xA3\xDE\xC8\xAC"t\x8B\xCAr\xE6\x82$8\x0F\xE1\x06\xF9\xBB\x0C\x9F\xAA"8\x94\xBBU\x8DQ\xC3\xE9@'\xF9\xBF\xCF\x8B\xD4\x995\xEB\xE9t3Q\x10\xD7\x0D\x9D\x94\xCA\x84\xC2\xE2{\x0B\x89r\xA6\xC6\xAA\xE5C\xC6U\xA6\xD1tqa\xA0\xD7RO\x92\xC9\xBE"\xF9\xD1\xDE\x93b\x06\xD3ae:\xB7\x8C\x99\xD6\xFFI\x86\x8CvtX@k\xE4m\x93wz\x8A\xC2U\xFBb\xA2\x9Ao\xAF\x8D\x19k\xA2pP\x83s\xFE\x0E\x0FY\xA0\xA7E'\xC0\x02\xF4\x98A5\xF2\x8A?\x04$\x89\xC7]\x0A\xFBX\x97*\xAEN\x13#\xB3a\xD2y\xD3#_B\xAC\x05:\xAC\x92\xEAO\x08H\x88N\x1A\xB9\xDC\xFA\x11ikJ\x12!\xE8u\xCD+\x88\x98\xE3c\xCB\xD9<1%\x98KDN\xC6\xF2\xB7\x86o6\x91P`\x9B\xA1\x0B\x82\xEB\x15H\xA0\xC4\x85\xDF\xAC"\xA1b\xD9\xA3b\xB8E\xB59_\xF4\xCCWH\x09&5\xE6\xBE\xF2\x19FC\x0E\xAB\xEA6\x0C\xAD5\x90/$D\xB3\x9E\x81[9j\x8A\xC4\x85\xAAA\x7Fe\xDCG8\x00\xDA\xCFi\xBDp\x18?\xF5\xA8~@\xC1\x08\xDF\xE5\xAE,\xDF0t\xCB\x92W8&V\x01F\x1A./\x8D\xAF\xD8\x87\xCE\x80w*\x18Is\x17\x15\x17DI\xB4a_\N\xB77\xA7n\x16\xDF
IE\xEF\x9E\x8Cd7\x1B\xF9\x97\xF9E\x86\x98\x9F\x1D\xB6\x9F\x94\xF7\x8AJ\x1A\xCD\x88\xD3\xD3\xDEw\x92Q\\xF5\xC6\xD6\x11c\x81\x00\xE8\xD9"'\xE1\x9D5\xFC\x11},\xB8\xB2V\xE6\xC0\xB7a/\x18~=G\xAC\x9EGxR,\x9B\x91\xA0\xE9\x85\x14J\xB3\xB2O\xEA3\xB2F\xA7vo\x88\xFEm\x18*g%\xA4l\x9B\xF0\xA5`$\xEBo\xFC?\x13"s\x0D\x91y\x92\xE0u\xFA\xD1p+)\xACpi\xE3\xB4L-\x0A\xF1#\xCF\x1A\x82\x8A\xE5\xEF80\xC9\x17z@\xD1\x9AoK\xCE\xE42\x92M\xEF\x85\xBE(z\x860\xC6\x03t\x02\xA1\xD2\x09\x1E\xB3\x80t\x86|\x8E~F\xFD1i|\x84^\x07\xC9Z\xBE\x91\xA7\x06\x9B\xC7\x8F\xFB\xD4\xB84\xED\xA2\x108/X\x89\xF4W\xF6\xE3\xEE\x94Q,H\xFF>o3E\xA7q\xE5\x15\x86\xCF\x0F_\xF9\xE8N\xCD}\xEB\xFD\x0E\x03EZi\x83 \xA7D|{]\xEE\xBA\xE4\x00RR\x1C\xFBj\x81\xF0{w\x9F\xA6F\xBB\x00\x0Cw\x01\xE1\xFE\xC8\xE8\xAC\xD18\x19,\xE9\x9E\xFE\xF0\xA45ov[K\x86UT\x00\xC0*\xEF\x9De\xE7pN[\xA5~\xF3\xDA\xAD\xE0\x85\xB0Nb\x09I_\xA8B:b\x9A\x10\xC2\xCF\xE83|\xB1\xCD\x17\xE8\x95\xA6!"\xD72DR\x03\xB7\xF4"\xC2\x88\xF1Rl+t\xA7x\x04\x10\xC1@\\xC3\xE5}\xDD`\xA2\x91W\x7F%S\xB7\x1
3\xFF\xF2\x82~e\xDC\xDD\xA1\xC5\xDF\xC6xx\xAC\xF3!\xDC\xFE)\xCF@\x8E\x9CU\xE9\x03m\xF7\xB5\xC8\xAB\xB9\x94\x02\x8C\x910\xE6\xBC\xA3\xFAc\x18\x83v\x09\xAF\xE9u\xE1\xB6N4\x16\x8E\xD0\xFA;\xBF\xDEt:\xDAq\xC6~\x979\x0C\x9C%=\x19\x09\xFB\xC7\xB2,yV\xED\xC6}\x0A\\xFB,*\xEF+V\xA8\xA9\x81\xADOD\xF4\@\xFE\xAEV\xED\xC8\xC5O[\xE6\xC7\xEE\xC7\xA0\x10L\xFFiY\xAE\xD4\xDE\xE89\xF6\x83\xC8\xF7\xB9-\xB8\x0A8{A8\xF2k\xF0U\xADh\x1F\xB5\xEF\x14"\x8E<U\xFE\x93\xBD\x15,\xD3\xC3\xB3\x0E\x09)\xE4`\xDA\xA1\xA7n\x96 l\xC4\x93*\x1A\x8Fx\xCA\xDB\xE6\xAB\x8F\x04\xED\x0E_\xF9\xD7\xDBK\xCB\xA8\x15!\x06I\xB5\xA2\xAA\xC2+\x99c\xE3N\xC0\x0E\x03;JIIu\x8E\x9B\xDB1\xAB9j\xDF\x8B\xF0\xBD\xEC\xA2\x9F\x11\xE9\xF7\xCF\xBD{$\xD2\xD1\x94G\xBA~\xF2\x8B\\xBC\x9E\xD2\x10\xB5\xF3:\x00\x8AK\x87~\xC9\xFE\x8CJ\x1B\xEC\x1D{S}?\x9Fu\xD3\x18\xDF\x94w\xB2\x85\xB5Mx\xFC\x18\x02k\x14\x83\x10\xACX\x1B\xB4\xA6\x0E\x0E2\xDByn\x15#\xE0\x9C\x01J\x85\xF0\xC3\xC0\x19,\xB51\x9E\xD8cPV\xC2\xD1\xFA\xDCV\x13PNV\x10\xA1\x18iaP8vWY\xCF\xD0\x80\x0DW\x
1E\xC2\x81\xF2z\xEB\xAE\x85Z\xA6\xD4\x93\xF4\xB7\x0F\xF2\xA2\xB2tw\x96\xD1k\xB2\x17\x97F\x97\x84\xE5)\xD6\x0E\xD8\xEFN\xB9y[\x05\x80i\x90\xF6\xBE\x18yRd\x1FR\x02\xBA\x13q\x1E\xF7\xE4\]\xF2\xE5U$\x7F\x9DFe\x04\x10\xB4.\x0F\x9B9\xE8U\x1E%\x15TL;\x9C\x8C\xCD\x89\xD9s0/8\x15\xB1\xDB\x0D{\xD8/\xBE\xFE\xB2\x17\xD4\x05\xD0\x1A\xC6\xAD\x09\xB7N\xEDBe\x009'\x11cJ,\x89#y\x7F\xEC\xCF\x0F[\xBF\xB3\xFE\x1C\x88\xC4\x04\xB3\xF5G\x8C\xAA'\xC1\xEFT5\x19\xE3%\x8EI \xD7VB\xC1\xB1\xD8\x02\xCC\xFF\xBE\xB4\x8D@\xDF}\xD2\x94\x0A\xC2\x10\xE6\xD0\xFA\xDB`\x00\x91\x1D\xD6\xAE\xC9\xFE\x93\xC4e"?\xBD\x8EH\xB6\xCD!\xEA\x9AB\xBE\xD9\x8CQ\xBD&\x8D\xF3oz&\xBE\xACqF\x8E\xF0\xD0\xD2\x19r\xE5s\x00f\x9B\x9F\xA3e\x89V\xBEP\xA8)\x87D\x14\x83\xB1eW\xAB"\x95\xDAQ\xCF1\xE6H\x10\x08\xE8\xFF7\x12\xE3:\x0D\x02\x8F=2Dp\xB1\xEE\xB6\xF03\xB6\xAA@\xAF\xCF\x8EJ:V)s\xB9\xA9/\x1EURY\xC0\xA1o+X|\xFA\xB6\x87\xF2\xEBk&[\x07\xA7\xC90\x90F%\x13\xAB\xAF\x11N\xC6\x9F\x0BV\x1Db\xD8\xD4\xE74\xF23\xFE\x84\xE9\xA3\xD6\xF4\xF5\x19\xB8A\xC8\x01\
xD0B\xC4\xFBN\x7F;\x7F\xFD\xA3HT\x84\xC8T\x18W\x7F\x9F\xDC\x0Dq\xB2U\xA9T\xF3]xx\xDAk\xC1\xBF\xF4)\x8Eh\xD2\x8C{\x90\x05_\x1F\x03a`\xEE\xD5Wx\xBB\xD5\xA5\xF1m\x17\x85\x97:J\xDD\xE4\xE2\xB7t\xC4\x98x\x99\x86\xE5)\x82\xE1\xEDT\xB1\xCD\x0A#C\x95\x16\x85\xA8\xCB\x89\xCDh\xC0i\xC3t\xD3\xD6\x9Ez\x8D\xEC\x87\xB9\xE4\x85e\x95\x91VG\xF79u0C \xF6AFBK\x11+\xB2i\x8F\x87c\xBF`/]Y[>\xB0Cl'\xAE\xC6Qn\xD4V\xB6i\x07\xBA(+\xEFZ\xE7\xB2\xCC[p\xC0j\x90\x8C.l*\xE2\x97;&\xD9\xB7\x8D\x8A\xE5S\xCEtT|\xAC\xAEp\xACg\xF2\xFC\xB5\xECac\x0E]\x09,\x09n>#\x1Akg\xC2\xD5|\xB9\x8Df\x17\x851v'|*=\x8D\xFA\x00>\xB2\xD9\x04&QI\xB0\xC4{\x18\x1A}\x8DV:\xA8\xECN4n*\xAE\xE4\x07YK\x03\xF8ms\x17/\x17J\xFD\xFD\x14/\x86X\xCE3\x1C^\xC4\x8D\x9Bi\xF1\xCA\xE6\xB2\xE9\x83\xFD\x0F\xCD\x8A\x9F\x1D\xB5\xC70\xBAU\xFD\xDCOR\xF7Y2I$^\xBD\xFC;"\xE6\x10\xF4\xA5\x80\x8B\x13\xF7'm}\xD3\x8C\xAB\xED\xE0\xE7\x1E\x81\xD1\x1B\xC3\xCA\xCD\xB1\xD4M\xD7}w\x85\x0A\x00a\x94\xE5\xA9\x96\xD0\xA2\x19\x03\xFD\xDAM\xC8w,\x11!\x87\x0D%v.k\xAB\x88<\x12\xE6\xE
1<\xAA-\xF6Kb\x03\xEFt d\x8B\x1A\xB5\x08\xD1\xF6\xA3i\xDE\xDF\x90s#\x88\xE3\xE8\xA6\xE7n&\xFEv\xC2\xF6\xA1\xD2\xDC\xE6\xF1%H\xB9W\x86\xF3\x8D\x8AO\x10r\xD3\x89;\xD1\x90(H\xA6ei\xFB\xCB\xDB\xBA\x93z\x01C\x1B\xB0\xA2p\x04\x80\x01\xEA\xA0N\xF9\xE5\xE4\x83\xED\x0E\xBCM\x06\x8E\xBE\x14\x8A\xF5NJ\xA0\xD0\x15\xEA\xDBHJA\xE7\xEEyT,\x9E\\xE01\xC0\xA6HQ\xAA\xF6\xBB\xD2\xD0Y\xD7\xD2\x18\xC2|\xDFp\x12\x0B\xD1Ly\x7F\xD4\xA7\x02\x83\x18fm~\xB9\x02zAq\xF0\xF72\xE0k.\x872\x8B\xF6\x1C\x84V\xC8\xB7W\xF7u\xD4\x0A}\xF3\x8F\x05\x1BX\x86vh\x11Bt\xF3b\x8CX\xD3i3\xDA\x1E\xE0JHu*\\x0C\xA5\xBC\xAB"7\xDA\xAFdqz\xF90\x98M\xF2\xEB\x1D\x90\xEAY\x8B3\xD9\x13\x96g\x0B\xBEV\xC5{O\xA6\x8C\xF6(\x03cO\xDBAE\xFF\xA2y\xF4L\xDB=?.\xBB\xB0a\xEF\xECiO\xC6\xE7\xDF\xD2*4dp\xAB+s\\x8B\x15\x00\xF5a\x19\xEC\x0A \x0Dz\xD4\x80\xE0%\x97\xDD\xB0\xCB9\xC8`\x01\x0B\x93\xBE\xC4\xB2B\xD6Sa\xB8hI\x12\x0D\xA4\x80\x85\x93\xBA+\x95\xCA\x9F\xD55\x803k\x8E\xC1\x15\x14NsrhL9\x1E\xE9\x87\xDCE\xB5\x97\x96\xD6\x197\x9B\x81\xD6><5\xBE\xAB\xA9\xEB
\xCB\x85E\x8D\xE4[\x9B1\x9B\xCC\x90\xFFc\x161|QU\xB2\x18\x93\x1E\x84\xAB\xB8\x9F\xC8G\xB6\x05\x9C\xDD2\xCFM\xC0\xAEhsj\x12\xA3NI\xA3\xA4\xA6Kt\x0D\xBA\x86\xD1vm\xC3\xAD\x8D\xA7\xFD\x1D_\x92Uw\x1D\xB3\x82\x8C\x87r\x96\xD2\xF8\xFAy\x91C^G\xCE\xD1-b\xE06\xCC|]\xB5\xD2 \xEE\xB97r@f\xB9{\x18\x90\xA7\x08\xD6\xF7"\xAC\xB1:.\x08\xE6Z-\x12\xFB\x01\xE6\xE9\xF0@\x89S\xD7\x06,\x7Fy\xC6\x83\xAEG[3\x0CO\xFA\xD7\x92\xAD\xAC#\xAFgo\x00\xB4\xA6\xE1X\x02\xC8B\xD0Tc\xA9\x86\x1F\x07\xCE\x98\x92\xBB\xF1\xCDR\x9F\x9B/\x86\xF6\xFB\xFA7\xF2\xF3\x90;=\x86\x91y\xE9%\xEDTJ\x90MH\x07le\xBC\x0E\x8C<\xB3^\x0E\xC2\x9F\xCB\x18\xA7\x9BL\xE7\xEC#\x8C4U\xE5&#\xECYn\x0E\xFE\x0A&\xD4\xBA\x15\xDE\x13\xE17\x0Bl.\xE76\xFA\xCF\xC2,b\xC5\xAFh\xF9;4/\xDF\x1B\xB4.\xFD\x0A0\xD1|K\xA6\x90**}<\xEC\x94\x8E\xD3cMH\xFA\xAC\xF2\xB9[60t\x09\xE3\x8E\x82\x05UU\xA1T\xC50\xEC\x1F\x99 `t\x12\x18?o\xC2\xB0\x0B7\xA1\xEC\x15\xC7\x1C\xF3\xE8\xB1\x95\x9CG\xA1U\xB7\xA4)\xEAe\xC4br\xDA\xFC\xE6\xEAdu\xD8Y\xBB\x93w\xD7R\x80&7\x96r\xD3\xBA`\x8D\xEC
\x8A~\xE6\x02\xADK\xE3\x9C\x1F\x07\xFD\xD3_%\xEEk6\xE5@\x15\x96g\x1FN\xB2\x03D\xD7?9\x9A\xB5x\xEAS_}%\xDDT\x05i6K\x8A\x8D"3U\xD7O\x04\x10\x8F\x93\xC8\xB2u\xE9\xC4MJ5\xE8z\xB0\xC5\xAD\xFD\x9C\xBE\xB2lT\x99v\xA2\xD0\xD4\xA5=\xF0\xABwF\xD6\x97\xB6\xA4\xFB\x9E\xAC+\xFAz\x8BtxzKx<5\xE9\xE33o\xC3\xDC\xA7\xAC`\xD8\xC4\x91<\xCBh2\xD2\x1E\xF8\xF6\x95zv\xC1\xF5;_a\xE76\xF4GkV\x04\xE9\x84\x12\xD0\xE3b\xA6\x99\x01\xF8\xB4\x1Auw\xA3\xA5\xD0,\xB9*\x8D\xE8Nv4\x14\xB6\x9A\x9F\xF0\xB3\xD7\xFD\xE9\x8D\x8B\x02\xA5\x9C\xE8\x01\xB0Q\xE0\xCD\xD4D\x8E\x9E\xA6\xCAL\x95\x86=Ac\x13^e_\x8A\x1BR\x99`\xB2\x1E\xAC\x1A\xD0\xE2\xB1J\x87U\xE7\xA4\xF3\x11\xFD\x99qQ\xD8Ug8\xCD"\xC1q\x8DJ&0\x1A\xC0\x83S{q\xCE\xE9+\xED\6\x19\x14\x0A\xE2\x98\xC2Lv\xC0\xBF\x19IL\xD7^?\x0B@\x1B\xBD \xD5?\xF5\xEE\x1E\xF8\x128d\xDB\xE7h\xB3v\x8Ae\xEB\xE88X\xC1\xDBT\x0FN\x932\x1E\xC9\x8C\\xAD(WVf\x8Fp\xAA\xCE\x00N\xEC\x19\x9B\xC5\xE3#y\xE8\xDC\x8E\xBD\xB6\xAE\xC4>\xA8\x12\x88~\x1D\x01\x8F\x1C\x13\xCC-\x88A\xAEj\x92\xF0q\xCE\x1E(\x9E8O\x16\xE
8\x8C\xA3\x98\x9F\x948\xF5\x8D/9\xF1\xB8!\xB7 0\x9B\x0E\x1FQ,iB\xC37\xB5\x85\xBD\xD2\x08w\xA8\x18\xDB\x7F\xE4\xEB\x93\x9B\xB1X\xF4\x9C\x1C]\xB8\x08]:f\xD1z\xCB\x89\x89\xD5V\xF5\x92\x19b\xC0\x10\xAA\x0A\xA9\xFD-\x16\xE3\xB1tm\x8E>\xE02\xD2V\xB7\x09\xF8\xAACZ}\x8C\xBCL\xCB\xE5\xDD]\xDF.B\xD2\xFD\xBF\xA0\xA8\xEB\xA1,V\xAA\xC3J=\x1E\xD3\xA1\xC64\x98{\xD7f,3\xBBF\x00\x09[\xB0\\x07\x0A\xE0?T\xBCr\x13\x17b3M\xB3\xBD*\xBA\x05J\xC4G%\x91HT\x18\xB5\xA4U\xF0r\xAD.T\x1C\xE5e\xD7\xF8e\xFD\xD5\xBFE\x0B\x08\xEB8Q\xD1\xF4\x93\xA1\xF7V\xA3\xA5\xED>w_\xBA\xF9\xC47\xC8\xF7i\xF2\xA9n+^\xE8\x85}\xE3L\x90\x86\x8Dt,\x96C\xDC|\xD9eX_\x0C\xF3\xE3\x84\x0E5\xC6\xD3}\xBB\xBFA"\xAB;K\xEE\xB1\x14\xFB\xBC\xC6e\xE7D\x03r\xAD\xFFr\xA3\xBFv)\xBAA\xAA\x8F\xF6\x04\xB1^\x01$\x9E>\x98OE\xFB\x09:\xDF\x95\x9DI9\x83\x8D\x8C\x05r=\xDC\xD9V\xE3\x9B\xC4\xD1\xC2\xE19\xD9\xCB\x9F\xE7\xEA04\xD8tj\xABQ\x13\x17Qj\xC4\xB2)`\xFB6\xC5\xDB\xBF\xD53\xF8\x90\xDA\xB6!\x95p2\x0B`\x18D\xA2\xDD\x87\xDB\xD4\x9FW\x92\xE4:\x9DB\xA0F\xC7n^\xF6]
%\x88\x9D\xB0\xCBk\xDE\xB26\x81A\xC6LrAG\xB3\x8DE\xE0\x04\x0DV\xBCU\x9E\xB8\xC8\x91\xE2\x08\x1A\x9E,\xFB\xFDvh_L\x9F\xF8\x9F\x98\x06\xCB\xE5\xA3\xF6t\x87\xBD\xDF\xC0\x9C\xB1\x84&\xC38\x1C6\xFA\xFE\xC3kE:C\xA67h\x81\xCE\x0AM\x0D8\xCE(\x13\xC8&\xF9\x85\xE4u\xD3FN\xB8\xBA1PI\xD3\xEB\xB8\xA6(#3})o\xFC\x93\x89h\x19\xFCy>U\xFF'\x833v\xAE\xEB\xDD\x8E\x12\xBC|\xAB\xB4\x1F\xD4vI\xB6\x1F|#\xA2\x87\xF0V\xA4'L\xCB\xE4\xBC\x9C\xDA\x97c\x94~\xEDw\x18\xAB\x8B9}\x8F\xF5#\xEAd\x9Dv\xD4\xDE\xF5\xBAV\x9Ey2\x1Ahg\x02\x94.\xD6\xEC\xF4\xB8\x13h\xD5\x14\x08g\xD8\xAA\xB5\x03\xF9\x05\xDF\x19\xABV\xB2\xB5\xDA\xB2\x06\xDC\xA1-+\x06\xFE>\xCE\x08\x17\x82eDe~p\xAE\xB0\x1B\xD1?\xCB\xFA[8~~&\x19\xD1\xC6\x97Ri\x0B\xF7\xC2\xA8\xAB\xCA\xBA\x0C6\xEC\xC9\x8E@\xAC?\xD1O\xA8:\x81\xD7SM\xD2\xB8\x12\xDF.r\xFFJ\xE1\x89\xA2h\xB6\x1C\x1D\xF3\x0F\xE3E\x9F\xFCT\xABH\x19\x84\xA1s)s\xF9\xDB\xD8^\xE3\xA2\xF42\x7F\xBD\xCB@\x14\xAEj\xB8z\x1F7\xCC\x07\xE7\x17\xD7V\xC8\xDEN\x8Fj\x0Ei\x17Yf=\xBB\xE7r\xE7?'\xE9^\x90\xF4\xD7\xADh\xBAC\xE
7\xF0\xCB\xDC\x8C\x11\x9F\xF1\x16g\x19N\x98\xC4\xF4\x08\x15N%\x9CUZR\xE2$W\x16\xF7\xAD\xB8\xAB\x85^\xECp Z\xC8\xFER\x96G~\xD9\xA4(6\xD6\x1A\x8B\xA5\xA8\xA3o\xE6bh\x93\xFAul\x15\x8C\x96\x12\xE4\xEE\x82\xCA\x92\x9CW^\xCF\xEF\x8FrP\x9B\x91v\x17}\x07\xDE\xD1C\xDF\x14\x99H2#\x9F\x1D@\x13cJ3\x8A\xE3E\x95S\xC35'\xB0\x12\xAF"\xC4\x98(\xF7u<k\x8E\xA5s\xD4\xE6\x10\x98\x8B9)\x08\x8B\xBE\xECH\xBB\xAE*\x05\xA0\xF6J\x04G\xED\x97\xCF\x98'\xBE\x1C\xF4\xB6\xAC\xC4\x9E\x0B\xF4 C\x87N\xC8^^g\xBD\xF2\xCA\xF1\xD9\x95\x88\xF6|\xDB\x82\xF1\x81\xB7vpjS\x8CQ\xF2\x81I\x93\x05w12\x01\xF3\x9E\x15\x88#G\x87\x80W\x0B\xAA\xAB+P\xCC5\xAF/'\x10\xF5\x7F\xCB\xA5\xE6+8\x99\xA0\xB5\xB6[\xED\xBE4\xAD=\xFD\x1F\x18\xECV \xBB\x9F^\x95\xBB\x7F4\xE4\xE3\xE9\x17\xE9\x99\x19\xEC.\xA3\x8F\x0Fq~\xAE\xDF\xA5\xEE\xAEJ\xC1\xD6uCY\x06\x1B\xFD\xAB\x88\xA9 \x8A/,\x13\x8C\xB9\x1CKH[\xDBv.\x90H\xF7p\\x97\xDD\xC9\x8Ffx\x17]0(\xE5\xE1\xF9\xC2b\xC9\xBC\xE1*\x8E\x0DpZv\xC6\xF5l\x8A\xD3x>u\xCF0f\x8AN\xB2aX\xEA\xD5\x09\xD6\xC1C\x98\xBC\x1A=M\
xFEz\xAA\xBDB\xE8\xCE\xDD[\x09\xD3\xDD\xD6a\x89\x1C\xA8?\xBA\x03\xBF\x99\xDDV\x08\x02\xEC\xBB\x81\xD7\xFB\x12\D\xC9\xCF\xD4,?\xE7\xDAS8\xC0~Z:C\xDB\xFE\xEE\x0B\xFA[\x10\xFBJU\xC9A\xC7I\x11\xAB\xC3\xDBoqC\xFE\x89&\x88\xB6\x9E}\x9A\x0D\xEE\x85\xED}MM\x99K;\x9D\x09\xA4Y\x01\xCA\xA0\x08\xDD\xC9\x05u\x87\xF4\xA8e\xE8 >\x89M\xE87\xB6d\x96W[\xE0\xE7\x8C8\xD5\x83\xEA\x00\xB7=\xDA0\x83zp\xDF{\xF9L\xCE\xAF\x01\xE0\xF7\x8F\xA7\x9A\xAE\xB2\xE1G3\x9F\x90\xA8\xE8\xB0\xD7\x9BJ\x83\xC5\\x1D\x09\xA7\xBB\x85\xBB{\xEE\xFD\xB3X\xF3\xCDE\x19\xF5\x0F\x16\xE6\xDA8\xA9R\xE6\xE9-\xDCk\xDFj\xFC8\x15\xF1\x84\xCC\xB079w\xC8^(\xAE\x1A\x95\x90}\xAB\x8E\xCB\xCB\xA5\xE2\xC5\x99'\xC2\xCB\x89\xA7\xD4\x0DS\x89R\x9C\x86`\xE4\x10\xBB\xE2\xCE\x19P\xF2I\x13<\xE9\xE8!\xCE8p/\x93\xE7\xD8\xBE\x95\xC8\x0C\xDE\xF6\xAFT\x06D\xFC;\x9F\xD1\xABV\x1F\xE5\xF1P\x8A\xE8\x8DG\x87\xBB\xCA>f&\xD0J\xF6\xDA\x0D\xC4\xC2\x98\xF2\xB5\xFB\xA6I\xBA\xEAV\xE8\xFD\xE6\xC9\x1A;\x82.\xBF\xA4\x9E_t|\xC7y}\xEA\xF5\x98\xD0\x8E\xCB\xC4\xF0G\xB3\x9E}b e
\x7F\xFD\x1C\n\xC0{x\xF7KV\<\x1D;\x0F\xA1\xFDne\xA5c\x9A~yo\x90\xB9\x1C7`Y\xB1E~\x80\x95;\xCC\xA4\xDCZ~\xE0\xF9WH\xB9R\xBA\xFA\xF0\xEA \xCC\x93\x02\xA8\xA1\x1E)\xC4\x99\xB5p\xF8\xBA\xE8\xEA(\x0EF7E\xAF*6':n.\xCE\xA4\xB8\xD5\x04\xF1\xA2\xD3\xFEn\xC8\xCD\x94x\xB1}\xEC(Cr\xAA!Q\x9AU\x80\x12\\xCAS\xAB\x16\xDAEn\xBB4\x15\xD9P\x94\x18\xDBO\xBA`L\xA5,F\xBE\xA1\x08\xBDZ\xA5$\x0E\x12rE\x15Z\xF7\xF3\xAD!08\xD9\xA7\x8F\x1E\xC5/\x90J{\xF2\x0Cv\x0B\x8Af*\x8D\xC6-"\xD1\x89\xF0\xDF\x8Fi\xB1"\x9Ft\xFE{\x1A\xF6\xA5g`o\xD0\xDD\xD2\x06\xC4\xBE\x0EAx \xE6\x89\x05\x13\xC48\x03\xA2\xCA\x8Dk\xE2\x97c#\xCE+\x8D\xF2\xCD\xA9V\xC0\x11\x80\xE4\xEC\xA9uc\xE2\x9B\x03d\x88\x8F\xDA"\xAA\xDD6\xD7\xE0\xA5mm\xBD\xE5\x9D\xC4\xF3\xC0\x92\xFD\x10\x07\xE6(\xAF\x13\xB8M\xA6r\xD9\xAD\x92\x81\xC7_\xBB\xA44\x14\xEAO\xB7#N\xD8\x97\xAE\x1FF\xED;\xED\xE3\xFEX\xCD\x8A2U{\x1Ft#\x07\xA9\x13\x8C\xBCN\xA7.k\xB9\x07n\xC4\xA4\xBFF\x97zj\x85\xBFP\xB83(\xD0\x88\xF4\xFD@\xE9H\x93\x1A\x84\xD6\xE7\xF7\x99\xF7\x01/\xF8\x09\x85\xA8\xB2e|\x09
\xE9\xD6\x9E\xF9_C\xB2\\xD81\x12\x16\xD8L\xF4\xBC\xD0\xE3u\xFC\xF3\x12P\xF3\x02\x12\x0E)\x0E{(\xB3\x13\x8F\x95\x11 \x80V\xDA\xBA\xA8\xD8\x09z\x8A\x98\xCB\xAA\x02s\x81=\xF5}\xB8-\x95\xE1\xCE\x07P\x80\xD0OL\xB6\xCC\x8B\x15\x1D\xA7\x18\xE2X!C\x0D\x1CV\x13j\xFB\xEEs%\xA1'\x84\x89\x1A$\x96T'\xC8\xFB)\x9C\xE4;\x9DiV\xCB\xE3\xEE\x8F\x12\xB8\xC8\xC1\x01|\xB7m\xEE4\x04]b\xA3n\x1B)9c\x9A\xFD\xF6\x84\x15{`\x86-\x10"\x9D"\xF3\xF1\xF4CZF\xD6nR\x1F\xEBk\x8F\xDBf\x87\xF2\xD6\x16V\xA1?\xAD\xA0\x02s\xD0\xA8\xCA<\x9F\xD3k\xFA\xD5i4\xBC\x91$\x98\x04S\xD5'\xAEG\xA0\xBE\xA8Y\xA3\xC7D\xF7\xCC\xC6\xB8\xC0\xF5\x962\xFD9`G\x1D\x91qa}\x06\x1FoX\x14X\y\x82\V\xD7:d:/`\xD3(\xBEw\xAC\xF3i\x15\x81\x8E\x82p\xA3b}O\xD2\x97c\x14\xE7\xBA?\xAE\x96\x14QhLgp+8\xDF\x1DD\x90~k\xC8L\.\x8DBSI}]Z\xF6\xB6\x90W8\x87\xBB\xDBP\x90U\x00H}\x05\xD0\xF5\xA7\x97\xF0\xA5\xD8\x9FZ\xD3\xC34\x0C\xCB\xD5p\xCB\xFBO0\xFFd\xD6\xB5\xB6\\xA7\xC3\xD83\xABX-o\x9E:4&o\xD5\xA8&\xC3\xA3d\x7F\xDE\x15\xA3\xC5(\xBB\xEE\x7FG\xB5\xAC\xBE\xB7\x07B\x04\x8
Ex\xD2\x15\67\xB4\xDCw\xE7Q\x8F#\xBF;\xE2\x13\xEDw\xFC\x0A9\xF6\xEAf\x07D\\xCE\xB4y\x04\x02F:\xC8\x7F\xD2\x87\x0F\x81\xD9\xEB\x01)\xA5`pd\xB1\x7F|x+,\xB2\xBD\xF7,J^\x1D0n$%\xDC\xFD\x05\xCE6!+\xBD\xFEB\xD0\xEA\x8E\xDC\xB7\x86\xD5\xCEE\xBB\x9E\xF9J\xDB\xE3\xC5F\xE5\x8A1\xEE\x09\xBB\xC8D\xAAU\xCAP\xBE\xA5S\xF3\x14j\xBE\x09A9\x0A\xD9E\x0C\xFA&\xA4\xF9'^\x14\xDE5\xA9>\x1C-\xA7\x19\x10\xE9]$\xD5\x87\x90\xFB\xCFoE.H\xBC\xC6-\x0EVi%S]\x10#\x82iG\xD0\x8B\xE3\xB9S\xF1\x00\xD0\x1D\xF3\x07\x81\xE22\x8F\xFA\x92Ke\xAC\x1Et\x901Fij\xFDW\xD9=\xDF\xD1_\xD3-\xAC/\xB6FK\x01M\x7F\x82p\x82!Y\xE1\xFA\xDA\x01A\x91?]\xFC\xE3\xE7\xCA\x8E&8\xE3\x0C\x9D\xAA\x17\x12S$l9\xA9}\xBC\x95\xEE\xEF"-c\xD1\xED`$\x1C7\xA1r\xF3f\x80\x1DgLk\x12q\x88qG\xBE\xCE)\xBFL\xF1{>\xBD3\x14\xFD\x86\x9A\x0E{\xD1\x88\x1CT3\x86\xD6 {@\xE4\xBB\xCDL\xEB\xA3\x1Df\x90\x07\x927\x12"\xF7\xE0"\x19\x86$f\xA83\xD1\x97\xDA\x88\x00\xC5\xB09\x19\x1FQ[\xD9\x14\x9EO\xA7mL\x0E\xEAq\x0D&}\x1E{\xD6(\xD2%\xB9\xEAX\xEC\xD1\xC7RR\xEB\x9D.C\x15\xA9\xE4\x11
F\x90\x0A.4H\x93\xB6\x01\xEF\xF2\xDB\xB7\x83\xD7M\xE3\xCF\xCB\x81gkN\x0E8\xA7n3:\x1D\x8F\xFCGz\xF7\xA0\x0E#\x94\x97\x95f\x9Fg,<C\xE06T\(\xEC\x1CG\xF3\x13\xB2:y\xFE\x86I\xFF\xF1\x99[jw\xD0\x07qu\xE2\x01?Sf\xE5\x83\xC4\xD3\xD3\x82j\x10\xFB\x91\xBB3f\x9E\x0E\xC5\xC0\xE1\xC6NG\xA8\x9D\xDE\xB4\xC6\x01w\xE5\xF8\x92\x06\xD3>A\x1B\xAD\xC5\xC3Q-}\xC0\xC0\xE7\xE3\xE1\x1E\x94\x0DP\xA4\x1DI\xDCAz^\xBC\x89\x83\x1A\x94?\xA0\xFC\xAE\x0A\x17;ni>\xD1%Yx\xB8\xDCG\xD0\x10\xCE\xA6\x93\x91\xE0\xF6\xD2\xC3\xACr\xBC\xA2\x0BU#\x0E\x95\xB4,\xC1\x83j4\xA9\x8Ck\x00EC\xB49\x1Dym6\x97 S\x94\xC4\xD45\xA8\x81UM\x18\xA8\xC8\xFB?H D\x18\xA7pa\x8C\xE4\xFB\xA90\x89Q\xFC\xE2\xC8\xA1}\x10\xF6\xB4F\xC4\x1A\xDF;lc\xC5\xF1\xF1X\x10p\xAFot\xC6*\xA4!\xB7\x1CN\x80_\x1Db\x0F\xCA\xBB.\xE1\x87\xEF\xC8a~\x0E\xB9\x95w:\x88\xCFX\xA2,\xBE\\x86\xC8fx\x98\x8CG\xA2\xBE\xFA \xF0\xD0\x08\xA5\xF8\x99V\xEDv\xE5\x8DK\xE3n\x07\xD7\x10\x11\x09d\xB0FFG\xD9\x09\x04\x83\x83\x03\xF6PY5c\xB8\xC2\x09\x85\xC2\x8C\xEB\xA0\x08\xAC\xCD\x16<5q\xB9\xE8\
xFBAI\x0B8\xEE\x9B\xB8\x09\xA0\x14\xE3\xFE\x0B\xE3\x03\x9A[\x8A\xFD}\x88\x1D\x97^\x13\x8C\x9E\x15Q\xF0\xC7oS\xB0\x99\x17\xE5\xDD\xB8X\x1E\xE1:\x92\xFAnu\xA0\xF0\x8D\x10L\xF2\x0E\x92\#Qg{E\xE8\x12/9\x8C\x1AIox\xCFjyC\x82\xE9!\xAF\xE4?P*\x0EM\xC0\xE4\x1D(\xDD\xE3V\xAA\xD8\xDD\x1C\x935\xA6i\xB7k\xBC\x83\x8F\x96\xF0\x9F\xEA/\x97\x07\xAC \xF4\xC3+\xE2\xF30>\xCD\xD4ct\xCA\x917\x88O\xF6X>\x96"\xE9\xD1\xFA\xF4}n\xBE\xD6v\xFE\xBC\xED\xA4\xE0\xDB\xF3k\xC1\x86I\xE9zR\xC9\x7F\x15\xF6-\xC37\xB0\xC2\x7F\xA8\xDB/\xA0\xD6gf\xC2\x1E\x1C\x10\x84\xED~\x80\x82w\x19}%\xF7^\xCD\xB9a{,n\xA8\x85\xDB\xA4\x90t\xD5\x0AO\x97\x01\x96\xE8/\xC8\x1EfX\x9B3\xDBE<\x08\xA4\xFB\xEC=\xC7S\x8B\xBD6\xB5O\x93w]\xECx\xF6\x1F\xA6\xEDA\xA0vw\x8F\xB2(\xD2D\xC7hQ\xA3%\xF6\xF4\xC8E\x135U\xFC$\xF4\x12x\xD1\xE2f\x13\x18Z\x13\xCA\xEE\xFF\x1FXn]kJx\xFFwR\xC0d\xFA\xBFD\xCBJ\xCC\x04\xFF\x8A\xEC\xB3TX%_UT6\xFA\x93j>\x9A)\xC0l\xF4\x0Dp\xD9\xA4\xCC,Z\xD0\x0B\x03\x18\xBE*\x18\x98\x17\x15X'E\xA7\x9DQ\xD6C\xFE\xDA<\x18go\xFA\x83\x17@\xFEQs
\x1F\x1E\x0A\xD0\x9D\xC3mvBR\x1B\xCE\xCC\xD9\xD9\xF3\x08\xB2\xEE\x840\xC7`\xCD\x1E\xBC\xD2\x8E#\xB0<\xD0\x9E\xC8#\x88k\xA9\x0EP\xE9\x12\xD6\xCA\xF4\xF0\xBD\xBB\x144\xAD\xEA\x8D\x15\xE9 \xCE\x00\x84\xDE\x19\x9FC\x9D\xE8\x1D\x93\xA9\xFE\xAC\x04\x02s\xD6AW,\x9F\xFF\x0A'\xF3\x0B\x18\x92\xE2-*,\x9F8\xA2o3\xE9\x00\x8F\xA0\xD1m-(\xE8[$qOA\xF9\xC2\xCA\xC6\xE6\xFD\x15\xC9EL\xBBJ%\x06\xF7\xA4\x17\xDB\xE7\x02x\x87\xCA\xF6\xF8 s=G\x8B\x9A\x9E\xC2y\x7FB\x9DQ/K\x04Z5\xA2\xFC\xB8\xC9H{\x87\xAE\x11\xD0<\xDD\xFF\x83\xBA\x913\x8A p\x7F\xD9\xA5vP\xC0f\xBF+{r\xF0\x9BD\xC9\xAD\xD3\x1B,\xED\x0A\xB4\xE0\xAF\xBC\xAC\xC1\x02w\xC3\xFF$\x8C9o\xD6\xF6\x0E\xA3\xAB!@\xE0\x19ged\xD1\xFB\x1DD1\x19:\xC3\xD1\xCF3"\x88\xEE\xEBL\xBC\x86\x16\xC2v:\xCB\xE4\xA1\xBB[Vp\x1B\xEEG"M\xEAK\x92\xF8X\xED-\x83\xEDo\xA7\x16Y\xFE\xDD\xEA0\xCC\xD5\xB1\xB7\xFC2\x9D\xAEf\x8C\x8C\x03B,B\x06Cnt\xCD\x99M\xAC5_\xFA\xF0vb\x1D&9\x81\x1C\xD2\xE3-\xBDTF\x0A\xB9\x0D\xDE\xBCG*+160\x86\xB4j\xE6\xC1L\x0F]\x85\x96\x8F">\x183(\xFE\xB1&\x14\x8E\x1D\
\xA8m.\x187i\x9DJ\x0DNbR\x0C\xF9\xB3\xD9|&\x0F\xB2\x1DA\x8C\xE5\xEF\xD75h\xD6\xC6\xE0\xEE\x86\xEA\xA5ZK\xD6\x16\xE8\xBFez\x17\x1C*\xB8\xB2\x06\xDE\x13\xA1\x05\x8E\x1C\x18\xBB\x01Y\xE7\xBBYv\x86\x96MD\x04e\xE1\xAC#e}\x05\x95\xEEW\x9F\xE9\xA6\xFB@\x9E\xD6d\xCB\x0B\x1B\xED1\xA1\xA7fg\xEC\xA0k\xC0\xB0\xBC\xE4t\xF7+\x07\xC3\xC0l\xD2T\xE4\xD2\xCC\x03\xE7\xB2\xD5\xE0\xB2}\xCB\xC7$gh!Ga\xB3\xE2`\xF4\xF9\x08_c,\xA5\xCD,\x81\xE4\xCF\xAF\"\xDB\xAEnh\x15\xB1v\x96]\xEF\xE6rRU]\x09\x89\xEFd\x04\x84\x7F\xC6T\x1E\xA1\x96\x02\xE1\xEEz3s\x8B\x7F\xAF\xC5\xBE\x0B\x10.x\xF7\x10(M2\xA5o.#\x8A\xFA\xE7\x9D\xA9\x07C^\x83$\xC1<"\x07\xD9D\xCBd\x11\x85\x1A\xDA\x80\xB9\x11\x98\xFB\xEB\xBD\xDB\xC6N\xBF\x0D\xA1\xD83cc\xFB\xE2\xDFD\xAD\xAFd<A\x117\x16\x8D\x8BA\x07\x17s\x11\x09\x99j\x13T\x0DL\x1C"\xF3\xB4\xF1\x8A9\xBD\x81\x0E\xED\xED5\xD0h\x9D\xC8\xCC\xF6\xB0\xEA{\xCD\x95\x1Ak\Z*x\xB4\xDD\x04\xEB\xD5\xA4\x8D\x94\x02U"\xAF\x92\xC5\xB7\xDC\x87\xCF\xA1\xF6\xC5\xCB\xDC<,\xBCY\x02VN\xA0\xC8\xA6\xA5\xA2\xAA\x9A\xAD-\xEB\
xA9\x12\x94Z\xB8q\xC0_{\xC9\x0B\xD9t\x8FJ\xA1\x04\x8F-\xC1\x80\x9CO@-\xD9Y\x08\-\xB7\x95\xFF\xFD\x03\x0F\xC5\x01\x16\x1Fg\xF2\xE4\xC21\xA6\x0C\xE0\xFE\x9C\xD5\xDC\xD8\xAD\xAB\x90:0\xB4b\xFB\xB7\xA0\xFA\x08E*\xE0\xA9\xD9\x14?\x12u\x85\xC8"+\xECZ\xE6\xD1*\xCB*\x80O\xEC\x96j\x0D\x15 \x92d\x85P\xBB\x0AH\xFE1\xA6\x97\xF8\xC3\xB3gZ0\xBD\x18\x87\x03\xE8\x0E\x82%\xDC$l\xC6O\x16\xD48\xC3\x97\xF8UE\x9F \x94\x86Al\x0D\xA9\x05\x0C:Ixej\x82\x14MC'6\x8B\x8A\xC0\x02\x0A\xD9\xD5p!\xAE\xF6\x1EBa\xDC\x15X\xCB%y\xBB\xC0\x09f\xF0#\x92\xA1\xF1\x94\xB9\x98\x0D~\xDC\xE2\x08\xD5\xF0\xF3\xBC\x84(\x0E\x10M\x1E\x1A\xD17\x96\x12\x97\xAF\xB2\x9E\xCD\xAD\xCA\x98\xBBc\x9D\xF5%qDA2FVy\x9E9\xD2\xFC}\xDFI\x12\xAA\xD2F\xD4\xE7\xE9\x8C\xC8\x86m\x83\x14\xC6\x15}\x15\x0D\xB3\xB9a\xD7t\xB8\x12pTi.\x0F\xF3i\xB3\xA9\xDF\xA8\xCC\x87\xC1\xA0m-*\xD0\x19\xBD\x9D\x11%\xF9\x99*\xAB\xDA^\x8C\xED9\xFD\x01\x96\xDF8m0\xD9\xE3\xA6\xC4\x9A\x80\xB0 \x83\x91)f\x1C\x05\xE1\xA8\xF6\xFD\xCE\x96\xAA8,\xBDS\x84\xFF\x05\x97\xB2L\x1B\x9E,\x09\
xB4Y\xE4\x02W\xD2\xB2\xA5\x19\xDE\xECL\xBAK\xBCT\xB6\xBB\xE7\xC9V\xEB\xDATuc8\x82\xD4\xF7F\xEA\xAD\x9D\x9C\xFD\xC0\xEB\x81\xC0?b\x03\xF1g\xC76\x1C\xA3\x07\x871\xE9\xC6\x16U|\x11\xF7\xCE\x02F\x87;7\xFB\xCD\x8F+\xABV\xB4x\xB6\x8FQS\x00\xD0\x84\xE8>\xE6[z\x8C\xF1\x13k\\xEDQ\xCDr\x84Y9\xAB\x90l4\xFEmbEK\xEA\xD9\xE5\x0EuI;L\x06\xD1\x94\xA3JV\x9B\x99o\xEE\xD18\x9A\xB0\x023q\x15[\xA6g\xCB\xB7\x92\x02+\xB7\xC0\x11\xED\x8F\x7F\x9E\xDAN\xB9\xA4\x12\xE0\xF4\xA3\x02\x12\x82fq\x07\xB9\xF2\xB48\xD7\xC7\x17\x8A\xFA\xD9\xB9\xFE\x9B\x157\xE27\x9A\x86\x15\x0A\xA3\x814\x8E\x07nO\x8AMUT8\x06\xC9\xE8`v0\xDE1\x03\xBE*K\xF6\xBF\xBF*\x97y\xE5\xD7\x93\xBE\x0C\xEB\xE8\xE7\x93\x84\xF5r\xE5P\x96\x9D\xECM@\xE7'\xB1+p\xF61\xAE\x8D\xE7xh\x91\xDB\xDE\xE1\xF7\xE9W\xEC\xA5\xDC\x00\xCA"SG\xFF\x96\x13\xBBU\xAF\x03\x09\x8E\x98\x99\xFE:\xC0\x90\x97\x94WG-\x9F/\x14\x09\x89R$4z\xF3\x86\xE2\x92L27\xC9\xE5\xFB\x92~m\xEB\x02\x1AN*\xB6\xD5C\xA9\x7F\x04\x8C\x9B\x03\x8BSE\x92y\xAEj:\x13s\xB2'\x14\xAA~\xBD\xE0\xC1\x92\x02\xD3\xD
0\xA5\xB0\x9A\xCEF$\xEA7\x03\x1A\x92`n\xDE~l\xC36;\x97E\x86\x92F\x86R\xAF\x83\x9D\xECpa\x9F\xCE\x86S>uO\xD37\x92\x91i#\xF4\x04\xBA\xAAq6\xA8D\xCB\x03)\xB6\xC6w\x8A\x19\xA8\xFD\x84\xF4\x87\xEF\xC3D+\xD4s\xB3\Q\xCD\x04\x10\xCA\xCBWm&R\x86\xFF\x0Ck\xEE\xD4\xDA\xB5\x88\xB4\x0B)\x9CG_)w&A5\xF1\x1E\xEC\xDF\xD6LT\xC3*\x91\x9E(\xB6\xBF\xB5\xEA\x97h\x0F\J\xE00\xB0\xD1\xAD\x17\xF8\xE9\xCD\xFFY\x92\xB5R\xC1\x83^g\x03,.'\xA1vr\xEA\xF2*\x1F^\x08T\xC0\x83\x0E\xE5"\x871\xE0\xD1\x9C\xFB\xD6<\xC7\x83\x11R\x91\x94q\xDF\xBE\xDD\xD7\x11\xFAs\xB2w\xA5\x89\xD3\xE5$\xF6A\xA4\xD7V\xD4\xEFKKL c\xA2\xC6>\x9E\x99\x1C\xF7TG\xED\xAF\x1B\xCCM\x1D\xD6P\x94\x07\xB2\x13\x80\x95b+\xBDRab\xD2,\x91\xC7\xF6a$\xD4\xE0LM\xF8\x8BM\xE4\x1F\xF3\x97v\x1FBv\xD3Z\x8E\x075G8M\xD9\xA2["\x90\xE31\x14\xF88H\xAF\x02\xF7{\xB1\xD1B\xF4\x14 \x06\xE3;\xB0bKX\x95y\x84v\xD1\xA7\x07\xCD\xF5\x92\xCFc\xB7\x94$m?@\xA6\xAD\xC2/\x08\xE7\x00\xB3"8u\xD2\x1CEv\xFBu\xA6\x0B\xC6{?'\xD5\x10\xE6?@\xE1\xE8\xC5\xA1\x09\xA7|\x11\xDB\xD3Y\x10Id\xD6\xDDW\
x8B\x04r\xCE\xE7m\x1C\x9Bb\xE2\x12R}2\xE5\xA6\xEB\xC9d\x80D&\xDBfC\xFF2\xF6*\x85\xD1\x8C\xF9\x1E\xC1\xA2;\x814\x00\xB1\xB5t\x07hzR\x83zs\xC8\xA0\xDF\xCEp\x96\xEB\xDE\x98\xFDh\xBD\xAF\x07e\xD6\xAD\xB8\xF3\xF62\xDA\x9E\x83\x89}\xE9\xBF`\x07D`LB\xB2\xE2_\xF5\xA0q\xB7\xFA=\xBA\xDC\x17O\xF2P\x06}\x1D\x98\x0F\xE1\x06=\x98B\xF3\xE6\x0CY!\xCBq\xC2\xB7'n\x8BK\xDAxL>\xDB\xF0%nY\xBD\xAEl\x90\xC8\x80G\x1E\x0A~\xC7\xBE\x96\xC7\xBAd\xDE=\x06\x12Y\x1C\x92Q\xC8j\x0E\x196\xF5\xF8\x1E\x8C\x15\xFF\xD4\xC2j(\xF6\xBD\xC83\xD9\xFC\xDA$9\x09Mj\x9B]\xC9\x7F\xF0\xDA\x0Es\xCB\x83\x0As\x84\xFE=\x12\x1C\x11\xF4\xDA\xADl\xAC\xF2b\xD68\xCB\xF2\xC1e\xE3k\xE8\xBD\xDF\xDB\xC1\xF0\xBA\xDB\x86\x8At\xAA(\xB4\x88\x19(\xA0g\x85\x99\x87\x03\xF9S\xCE\x10!\xBA\xCBU\xDF7k\x95\xA8\xA6x\xB4\xE3u3\xC8q\x05Jg\x0F[N)\x9An\x0B)\xED\xFB\x85\xC8b9=\x0A\x1EA\x1D\x15\xCAY\x8E7:\xFD\x91\xEC\xCD29Z\xEE)\xD7Q\xBD3]l\x9A\xFA\x09\xBB`<\xC1,\xC5\xEF\x80\xEBP\xD0I\x9B\x0F\xF0\xBD\x93\xE7\xF5\xDF.\x06\x03\x0Ca\x9AY\xB6\x97\xC9xk&\x98\x88+L\x
1F\x82V2\xF9*\xD5=\x9APL$\x0F\xB1,UFZ\xFE\xB2-S\x03\xB7%\xAAc9\xF5~2\xDE\xEC\x03a\x1E\x89 \x8A\xFFR\x81\x84:\x7F5Ff\x0E8m\xEF\x8A\xB7\x0E\xA2\xC0\x85\x07\x9A=\xB7\xE1\xBA\xBC)\x02r\x7F\xE5\xF5\xC6A\xB1\x0E"\xEF\x9A\xA2\x9B\x8E\xA1:\x98\xF3e;n\xA7Z3\xE1J\xAE\xCB\x9D\xD4\x0E\xD8\x0A\x12\xEB\xF7\xFBY\xD5A`\xF5\x86\x89Q\xB8|\xD4\x80=\xA6\xAD\xF5\x86\xB2\xA7\x1A\x15\x11\x9C\xA6\xA9\x97 S|\x0F\xA1\x0B\xE7\xC0\xEF*:\x0E\xF2\x8B\xBE\xD1\x8EP\x89\x7F\x08X`4\xCDF\x8Dn\x93\xD9\xE8z\x9C\xCE\x9D\x19\xD2\xE7\x08/\x0E\xEBy\x99\xC7p&\x8C\x07\x02I\xBF\x91yMyY\xD9GZ\x04\x9F\x0F3*\x91\xE8\x99\xD9\x85@\xCF\xE3\xC2Wx\xE4\xC9g\x1D\xCEf\x96j\xA4\x7FU\x05\x9B>\xF7I!D\x94ZW\xC5\x9F\x8Az]\x0C\xA7p\x01\x00\x0C|\x84\x1D\xA3\xB3\xA0\xD9\x93\xD6VQ\xD6\x06N\xB6\xB7Ix\x9E4\xB5)fI\xD6\xEC\x94Z\xA1\xAE\x9A\xB3A3\xC6rZ\xA8\xBF\xCE\x1C,\xE0:\xBE\xDD"S\x8C\xCFOd\xD3\x8D\x1C\xC4\xAD4i.}\x05\x8E\xE4T\x8E\x14-\xD6\xC9\x80\xBC\x82\xF6k\xE3'.HKCA\x9C\x14bX\x10\xF3N:K\xD4\xE6\xF3\x90\x18\xDE\xED\xCF\xBAo"wj\xC5\xE9=_cyY\xFB\
x86\xAA\xB2\x12#\x03z\xFAL\x87\x0CU\xB8\x9D\x8D{S\xCED\xD5\xEA(\x05}9\x01\xA3wRl<\x84\xB2K\xF5\x9C\xFF\x9CK\x8D\xD2\xA8z\xA9\xFA\xD7S\x83\x8FUE\xFC\xFC"\xCC`\xEC} \x1B0\xA7\xE3\x1C]G\x82\xE8\xCEDOk\xF7\xAE[\xD4\xB9\xAE\x1B\xDB>\xC7\xB9\xFB\xFF\xAC2\xFC\xDC`\xB8\xCA0?y#\xFA'L\x9F\x17\xB1\x95\x14\x03\xAFz\x8F~\xDF\xD7L\xAE\x0E\xC1bL4;\x9B}\xF2=\xBBt;Y9\xC9W\xC2\xD8\\xAD4\x0C\xF0C\xC4\xDD\xC6]X\x92\xAC8a\x8E\x12\x80\xAA\x0Fn>\xBA\x1C\xA8\x1A\xDC\xF3\xDE\xA6\x09OU\x95UD\xD4\x07s\x14\xEF\xCF<8\x07\xF8\xC7)\xFB'~?\x82\x91\xE09\x0AL\xF1\xEB\xD5\xCA\xC4\xD2\xC1\xDB\x93\x8D\x0C\x1EL-\xDD\x0F\xB7\xFC$$\x1BFR\x13\xB3\x9B\xB6\x01\x93a8\xB4\x896'\xDAl?v\x84\xAB#*\xAB\xEBsh\xAC\x1DTc5\xD9k\x02\x05\9\xA9\xE4v\xCC\x05-\xD2\x8F@\xD29r\x8E\xAAw\xB1\xAAX!\xE0\xE7iN\xFD'\xCA\xC9G/\x11\x08\x87\x8BT\x99\xF0\xDD\x01>\xC3-\x13\x9B\x05\xB7\xFB@ft\xD2\\xAF\x89\xBC\xE0\xCF\x0A%~\xD5\xD1\x1Dy\xD8E\xB2#\xF5\xAAsa\x1A\x12\xE0?\xD3\xE78\\xF6\xE8J*\x0F\x17P(.\x17\xACa)Rg\xAAB\xB3 8\xFE7\x9C\xDC\xBD\xE1\xAF~\xEC\x
B7\x18\xA1#\xF5\xBE\xEA\x8E\x83\x07\xC5\xE7iZ<\xBD\xFC&\x82q\x06N\x18E\xF5\x93)c\xA4s\x8A\x9D\xE2\xFE\x95\x9F\x8D\x95-\x93\xE8\x91\x83J\x8B\xF9Z\xDFB\xB0\xE5\xBD\x8D\x06u(\xC9~\xB8\xCC\x17\xCD\xB9\xBA\xB6^S\x15\xCF\x1A\x0D\xFC\xC7~\xF4-\xE3;\x1A>o4w='1\x96\xD1\xC6=F\x80\x10\x00\xC1\xEDE.\xD8\xD6\xFBi+\x05\x98kc\xD4h\xDA\xF3\xA6\x0F\x15\xDD[\x05Hn\xB3nG\xBC\xE5\xC5*\x86\x9A48\xC3SW\x19\xA5p\x8C\xEEw\xAC\xB3*\xEC\xC9\x82\xBD\xAD!\x9Flh\xA3<bZ\xE3T\xAEt\xC8=\x9A\xB9\xB7\xAC\x16\\xAC\x04y0\xBA\x84\x8F\xE3!{\x89/\x000\x8C4\x81\x1A/m^<'\xA5\x9E\xAF\x93\xAE\xC6\xC7\x93\x11~\x17\x0Ag.;yu>\x82\xFB\xBE\x158\x94Ci\xEB\x83[\xB7\xE2\xF5\xDB\xA2\xE9\xA3\xE63\xF9\x02!\xFB\x11\xB1I\xA6_\xF6\x86]sCU\xDB\xF8\xB6\xE5q\x02\x96\x14\xA7\x0EW\xA0)\xE2\xF8\xCF\xB1\x98#\xEE\x98\xEE\xDFp\x8F\x1B\xD4\xE6$\xFD\x14\xE9\x00\xF9\xBF<?\xA9\xF4(\xDBk\x0C\xEE+\xA4-\x0A\xE2|\x1EX\xF3\xE3\x8D}X\xBCN\xA16$\xCE\xA3{\x85xtN\x9CS\x18M\x0A\x95C\x05\xCCX7\x1E\x83\xB2\xC3~\xD6)ii4R\xE9L\xC0\xED$n-\x1C\xF3\xB4\x8B%\x9E\x98\xD
F\xF5\x0Es\x0Ckp\xCB\x0Dq\x8E\xBB\xE1\\xF1Y\x9AJ\xB9\x84\xFC\xD9M\xA0A\xDDs4\x9B\x06\x07\x84C\x12\xBB\x8C\xAA\xDDv\x8B\x80\x0A\xFD\xA8\xD3\x9C\xC9\x7F\xA8\x84\x8D\x80\xD7\xF6O\xB1[y=\xF1\xE6\xF6\xD8\x84p\x94\xDFQ\xFE\xB0\xFD'\xC5\xAA\xF6%\xAE\x8E|\xB9<\x1E\xDA\x16\xA6\x0C\xF7\x0B\xD5\xDD\xE6\xCB\xF6dXJ\x08fU\xDA\xF8\xBA\xA3\x85\xC9e\x1F\xA6*9'N\xD7\x86\x1ER\x90\xFEi\xEAM\xDC[\N\x08\xB4>\x8F\xB81*\x97EyE\xFD\xB3#"1\x07x\xD5!\x90\xB8\xA2\x11\xD1\xDBc(\xEF\x13\xD0\xE1\x8B\xE1~\xEC\x9D\xE9)s\xFD*\xFD\xD7\xFEFra\x9A\x9B^c\x0E\xD9Dx\x00\xE7\x93\x0D\x0E\xADy\x99\x08u\x00\xEC0 I40,\x17O=\x87R\x8Dm\x01x\x0B\x09<\x07h'\x8Dhf\x92\\xB4\xD1O\xFD{\xBFM\xE1\xE0\x18[z}c\xA0\xE7'<G"M,{\x86\xD1[\xF5\x14\x89\xE2+\xE8\xD9\xDC\x05\xAF9y\xDAJ\x80q\x09\xD4*\x93O\xB4>)|\xB2\xCA\xFB\xA3k\x83\x1Au@\x15\xF6\xE4\x18n|\x8Ch\x0D\x92\x9F\xE6\xEC%\x03}\x9Fe5\xD1\xA1\xC1\x01\xEA\x9CS\xE7\xFB\x07F\xA3\x95\xB1W7\xDANk\xBDX\xA6g\xB06\xC6,\xCB\xE9\xFF\xE2\xDBe\xDB\xA8P\xCC\xE62>\xBF\xC2\xD3T\xB3r\x810w\xAC\xD6t\xD2\xD5
\xB7c6j\x16n\xECT\x17Yx\xE74\x17\x9A\xC4:7}\xCE\x87m\xEC\xBE3\xA6\xA7\xC9\xDC\xEB\x16\x98&\xDC\xD1\xEE\x83\xA0<\xFE\x95KIa\xF0\xBB\xD6\x061\x87\xD4\xDF\x80f\xA3*\xF1z_\x07\x13\xE3 y\xBD \x96\xC2\xA8-u\xE9\xF2\xE6\x82|\x1B\xB0\x8B\x16\x81)\x04\x00\xAB$ \xA9\x81\xF2\x91\x8C\x04\x9D\xBAfh\xC5\x1B\xCD\xD0\xB5\xE0\x88F\x10\x8D\xAB'\x92\xB3\x9E\xB8\x10\x88\xBEN\x1F\xDA(n\x16[\xEE\xAA\xA8h w#\xEE\xB5\xD9\xBC\xAF\xD6\xAF\x8A\x98\xEC: \xF7i\x09gk\xEA~\xD6p\xD7\x89*A\x99\xF7{\xF3<\x1D"\xF1N\xB3\x08r\xEA\xCD33\x97\xB0\xFA%\xDF\xCA^r^\xAA\x99\xCC\xD6b1\x80`P\xDE\x99,\xF4\x8E\x87\xFD\xE5O\xED\xC7FPw,\x976\x15\xAA\xF4\x92}\xED\x0E\x9E!"\xD3\x80\x91\x84\x0C#6\xB6x\x06\xFCc\x088\xAB\x17C\xE8'\x1C1\xC0\xCDi\xA6\xF7\xC8\x06Ae\xCF\xBEvfa\xB6\xF83\xE0,\x87l\xEEzI\xA4\xD8j\xF23L\xA9v\x8D4\xB0\xA2Z\xE8;\xA8sV\x1A*\xDC\xE4\x18\xD5k\xD6\x02\xE7\xF4\xC7\xC0\xEBSP\x85\x02\x18\x1C\x19\x8F\xA0\x96:\xCFs\xA3l\x00W\xDE\xA3\x92L\x01\x07\xD9@\xB7m\xC0\xBBw#\xC3\xD0\xEA\xE9*\xB2O\x1F\\xDA\xC2\x0D\xB5\x85\x13\xFF\xD
8\xFC\x05E\x95g\xF8o*\xBFU\x87\x1E\xEE\xF9\xCE\xBA\xADm\x8E\xA2\xF5#\x0B\xEF\x9C\x9B\x0B\x04\x8E\xA44\x14Y\xF8Z\xDF\x06\xCA\xF0f\xD3\xB2\xB5p\x82k\x19O\x15\xB4-\xA7W\xA6\x90\x0F>J\xAB\x0D\x06\xDC^l\\xFF5_">m\x1Ee\xE5+\x0C\x81\xAC[\x12\xA8\xEA'\xFC\x8B\x19*\xB6I~\xB8G3\x07q\xEA:\x0F\xF3\xC7\xD4\x88\xCFWA\xFB\x04\xA9\xDC\xD2\xA3*\xAB\xF8AdQ\x85\xF5\x93\x94ba\xCC\x073.p\x02\x04\xB8\x10p\xBA\xC1\x07\xD5\x1C\xD1\xE3\xDA\xEF\xD4\xD0\x86p\xDE)N\xC3\xA2\x89\x1E2f\xD9\x92\x0F\xFD0\x81\x8B\xF8\xF3\xA1m\xA8\x9E\xB7}l\xBA\x0C\xEA\xF5\xD2\x94\xA1{\x98\xFC\x9A~.\xC5\x96\xF0R\x01\xA1=2|\xF5np\xA8e\xE2z\x05\xE4\x9E\x13\xD7m'\x11?\xA5[\x83\xA7\xBE\xE7M\xC6;t;.\xE0\xD6\xD15d\xE1K\x96\x0EUq\x8A\xD0Q5\x8B6\xF7\x03n\x81<\xBDv\xBB\x82d\x12>\xFD@\xC3\xAC\x94>D\x17\xA1\x8A\xA7\x11\x9AL\xBC\x91"N\x0F2\x8D2G\xBA\xEF\xA1\x8B.R'od|.\xE9\xB2\xD8\xFC\xB7\xCA\xEE\xF8\xA5o\xB6F\x85\xCB\xB4\xEC\x16A}\xBA;\xEDq\x8C-\xD1p\xFBiI\xF4w\x88&\x13b\x92d\x14\xA6]\xD3\x1F\x91\xDE=q\x84\xEB\x95\xDE\xCFfO\xDC_\x06.\x8B|\x1A\xE
1\xE5\xB3\x19vr\x1C\x8Cj\x11e!\x0E\xF9\xD1:\xD4K\xEF\xC4\xF3\xA75<)+\xFEn\x11_\xF7\xB4\x04\xFE\x19\x06W;\xAA+\xFA,L_\xF0\xDA\x9F\xC0G\xC3\xE8\xB1P6\xC6\x86\xA9]\xD3\xCB\x06w\xF1\xEC\xED\xD0\x0A\x98\xCDD\xE0+=\x9D8E\xF02\x07U\xDE\x8B\x8F\x10\x9CQ\xF0\xB6\xA9\xE4K\xF9t\x10\x9E'\xCCc\x87{\x12&e2\x94\xB9GuP=\x0Av\x10>\xE9\x96\xAB\xF94\x81=\xAFW\xC8+\x9Bj~\xDD_\xB6\xCEw\xBE4\xD6\xF1\xD9\\xCA\xB8\xE4\x1D\xE5\x1F\xF1]\xEB\x90\xC3\xA8\xCC'e\xED\x8At\x89t\x91\xB5\x0Da\xC2\xAF\xE1\x1D\xF6\x1F\x1B\x9F,s]\xA8!\xE9\xE08\xCD\xD9\x95v\xF1\x8E\x88\x85\xB0\xF77\x0E*\x9C\xA2v\xCA\x16\x1DE_\xE9\xBC\xFAu\xBE&\x88\xB3y\x1D*\xCD| [G\x8AI\x9Dr\x09\x8D?\xAD-\xFD\x88^\xE3\xBC\xA1\xC3a\xB4\x9Df\xE98\x8EE5\xCA\xC7\x95t\x83)\x04c\xCF6\x91/\x8FA^\xA7\xA9LJ]r\x7F\x00\xC6Yz\xA9'\x8B\xFE\x8F\x93\xA1\x08\xC4K5Ej\xA2\x9CT\x1F\xF2\x9A\x03q\x89\xE9\x1D\x8E~>\xFE\xB4\xAE\xCE\xC9\xE8R\xC8\x11\x13\xC1I\xC7O\x1Ba)GG|\x81_I\x10\xA7\xECx\x85y\xCF\x9F\x8E\x92(\x8F\xD6\x86\xF2\x09\xFEs\xA1\xAC\x85F\x84\xBD\xA1\xE3\xC1\xBC\x17
J\xB0:H\x05^\x8Bc\x82Y\x12\xCB\x81\x81\xBA`\xEE\xEC\xDFM\xC5\xBAKu\xA9\x8876\x95yl\xEE\xC1\x88vc'Zf7~\xAD\xC9\xF1hKi\xF0\x06\x9B\x8D,\x1C\xB6\x1F\xE7wx\x9B\x89\xF3\xF3\x05\xE9\xF9\x84\x09\x0B\xCB\xD0\xDC$X\xA3v9\xB6Myd\x14\xB1V\x87\x80\xBCE4\x1D\x0C\xE5.\xC8[\xD5+\x8E\x82r\x8C\xE4.\xF4\xD8\x1C\x1A\xB6#/\xB2\xE1]#9\x15<n\x93\x9B?k\xD1\x92?q\x8F\xC3\x1C\x1A\xCD\xAF\xD2\x19Q{p~85\xD7j{\x15\xA9\x84\x07\x9D\xACW7\xC3\xF7\x15\xF4\xFDa`\x7F\xD9\xCA\xC1Z\x15\x9EEF\x9D~T=\xED\x14TK\xCE\x9BH?i\xDA\x1FF\x0F+3\x96\xB2\x9A\x9A\xDC`\x06\xA6\x18\xFE\x04\xF5\xE1\x19\xA9\x18\xE0\xD8\x9D\xABy\xF9M\xC6\xE6\xC5c\x9E9\xA2\x93\xC1\xB8Jr:\x17\xFC\x81sG\xAA\xB7\xED\x1E\x0E\x9BgA\xC5h\x84z\xC1\xF1\x1B\x8C\xE7\xEF#rJ\xC1 \xB4\x8D\xE7\xEC\xACOF\xD6\xE7\xDC\xB9\xA3Qv\x82\x84k \x9A\xC6\x94\xEDS\xA5\xF9\xA2\x13\xEC\x10\xE5\xED\x93@\xD6\x1Br\xC7\x16\x9B\xFE6\x0AE\xC2*\xD4\xE7\xD6b\xE7\xC9\xAD\xE8\x11\xBB\x88\xFA\xB6|\xAE\xA2Q\xD8\xC3\x0E\x01\xA6#+\xBDI";)N\xA9\x89\xA2re\xC6\x15\xC6G\xC6I\xF1)/\xDA=\x13\x9A@y<\xF0
e\xFEgu\xD8\xC6\xFA_z\xBA\xEA\x1E\x81\x18a\x009\xAA\xFASvA\xB5\xF1\xFEi7\x9Af\xC4\xED\x8F\x9D\x84<\xAB\x1AaD#\xD5\xA6\xCB\xDEAH\x9B\x06\x09\xD7\xFC\x84;\x96\x9BJ\x7F\xC9+\x91\xDE\xAF\xAC\xBA\x156\xB4\x17\x98\x0D@\xD1NE\xC8CB\xDD\x0A\xAB(\x0F\xDA_5\xB6\x0E\xF5"Xs\xA2\xC4j\xB07\xB6Juzz\xCA5QL\xBD\x0Fa\xBA\x0AH|o\xC0\xF2\x94\x1F\xCE\x84s1g\x87\xE1S\x03\x151\x18\x937\x86{\xDA\xEA\x01Q*\x02\xC5\xEB\xF9\x136_K\xF8d\xE9\x1694\x02\xB3U\xDC\x10\xBC\,\x12\xA2J\xAF\xC0L-\xA6\x1CN\xB9+Z\x89\xCC\xC1\xA6\xECu\xD7\xEFj\xAD\xC5\xBCw\xBD\xF5\xCA\xF6\xE8\x92\x9A\x81\x0DJ\xF2]Pa\xEA\xCA}\x01xv\xFB\xC6\xD4\xC6CA\xD8\x0A"\x1B;\x95\x14\x90~\xEB\x81\xCAB\xF1x\xF9\xA7@\xA7\xC3\xE2\xC2N\xB5p>\xC9\xCB\x9Bl\x13\x1B\xC5\xA7\x1F\xE1?\x1E\xA2\x07bX\xD5\xDD\xCC\xF2\x93m\x84\xC4\x0B\xE7\x12\x8A_\xDB\xFA-\x84\xE5I6\xE7'\xDF\x05\xEF\xE1rC$}\xA7\xF7\xF2\xB4\x9D\xB7\xFCq\x9A$Z\x9Bz\xB0\x94\x1F\k\x1F\xF9U\x9F\x98\xEBl/#z\xD5fJp\xBD\xCE\xCE\xDD\x0D\xD8fL\xF5\xFE\xF4\xA2\x03\x0A\x0B\x08\x09A\xAF\x8B$\xE3Qq\xF2=\x0C\x1D\x
A1H:*\x18Z\xBF\xE4\xDA\x1B\x14k^\x18@\xFFs\x82\x90\x87\x8A\x92\xEF"f\x12\x12b\x134B\xE9\xC1\xAA\x0B\xF8\xCD/\x03=;\x01bc\xBA\x18%M\xAD{'`\x930\xBD5\xB1\xE4.k1@ M_|K5\x90\xF2:\xB9V\x9E=@\x97\xDD\x911\xCB,\x86Uo\xB4\x9B\xAC9\x9C\xCD\xF4\x00\x05E\xBB$\x9Bz&\xCBU\xE6\x93%o\xA5 \xC6Y\xF6\xD5\xF2\xF8\x98/6\xA6.\xF2\xC7kk(\x8E2\x14o<|\x8D\xA3\xF6\x82\xF2\x07\xF8\x1F2&\x1A:\xC8t\xE5\xFC\xFB\x8Cl\xF4\xC3\x93Q\xF1WGl\xCF5A.\x9Bl\x92ag(y(k\x8D\xB3\xA6\xB0\x00\xF6\xA7"\x83\x85\xFC.\xC5q\xD5\x815Q\x09\x0F\x12\x8C\x14at1\x02Q\xB9,\xBA\xAE\xF2\xFA\x16\xD9\x9C\x9B\xD1\xE2\xB5\x1C\xB0\x8AH\x80]5|\xFA_w[ \xD9R\xF51\x15\xC8\xBA\x91\xE6\xF4\xB5\xDC\xF8!\x1C\xBES{M\x84\x97m\x82\x17H\xDC\xF9\xF2\x81e\x82Il\xB9\xB4\xA2\x95\xC2\xA1\x19\xDB\xC8\xB9\x15\x15\x08\x07\x99\xCC\xCB\xB8.\x05(L\xD4:IJ|\xB4\xB5@\xE2'\xA3k\xAA\xD7%\xCC\xA0)W\xFA\xBD.:d'\xB77\x9A\xB1\x09\x89\xA5\x08a\xA5dP\x19q\x0Ey\xD0\x11E\x12\xD0\x09\xFAf\xAD\x07\x05\x17\xC4\xECA\x10\x0A\xE7Jx\x990/\xBC\xD7\xD6\x0E\xD8\x1D\xE8\xAD\x84\xE6db\xA0\xD2
X\xCF\x97\xE6\xF4{\xAA\xAA\xBE0\xA3\x17~QF\xC6\xDC\xB0<i[d\xB1*\x8E\x9E\xA89\x94YB\xD15Uc\x0Ec\x92\xB8\xA0\x10\xC5\xE4\xC0\xED\xEF\x89\xB6r\x93>\xFA[T\x86M@\xAB\x09\xF0\x9A\xB1\x07f\x0Av\xA6e\x9F\x93\xF6-\x1B\x19\xABMM\x96~D/d\x8DJ'$3\xD7]\\xB2s\x87f\xEC1=\x83#\x80~]\x9Dm_J\xCA{\xC7m\x1Fo>XL\xEB\xB4\xEA<(?\x83\x07\xF8\x00ksZX\xB0J\x8E\x06\xE1\xC7\x9BH\xF8P\x97\x84\xCE\xE9\x06\xF4\xC3\xC6\x8CK\x12\x98%$\xB1L\x89\xB7"\x110\xD7\x95\x92\xE7\xB0\xA4!\x80\xF5\xCA\xD4\xAD#\xC8o@\x03\x9F\xD0\x05\xE8\xDD\xBE\x0F\xA9\xBE8\xD8\x1F\xE5eG\xE9\xAF`W\x14@\xE5\x0F*\xE7\xCE.\xC5\x9D\xBA\x16^\x0B\xB7\xB6\x8C\xDC\x17\xB6-E\x9A3\x10\x81H\x8F8\xDC\x81\xEE\xB7\xE2*\xE1Z~\x08\x8B\xE1E}\xA7\xD8\xCA\xC0)3y\xAC@`%\x0CbP\xB6\xA0-'\xE32[D\xCE\x86I\x9D>\x17\xA5&\xEA\x16\xF3G\xC0>\x13\xB4n/\x0AJ\x01\xBAM\xF2\xB8\x11-\x09\xEA\xB8\x09H\xD7\xF7A\x08\xDE\xE9$_je\xAA\x96y,\x13\x9CKSD\xD2\xC14\xB4U\xB1tE\x95\xFE\x89\x8FH\xE7%j\xBB\x97d\x00\xF7\xD9g\xF45\xA2`\xDA\x8D\xF4p`\xA7\x0FWL\x0D=#I'\x8F\x05#\xEA}\xF0\xC4R\xA4\x
0D\xCB\x08\xFC\xB0Uc\x8EM\x95\x84!~v\xE5\xDB9\x14x\xEC\x11\x01v\xF0\xBC!\x8C*M\x08\x07\xC9\xC9\x9E\xFF\xCE\xA9#\xF8\xB2Q\xBE\x03\xDDS$\x9D\xD88\xB9\xC5\xC3\xBA\x81\x0D\xDBK\xE7\xD7\x13v\x9A\x1B\xC9\xA8\xAF\xC1\xFE\xA3-\xDA)\x98\xB9\x03\x08E\x9F\x93\xFC\x06LI\xBB\xD7B6\xEC\xC0verh\x96\x84\xEF\xC1\xDC\x07p\xBC\x85\xB00\x89\xDC4\xD5\x04\xF21\x04\xD3\xB2qA\xED\x83\xF0!\x96\xB8\xC7\x15\xB3q\xE0\x86 \xD0\xFE\xB8\x8D\x932\xAB\xCA\xC2\x04L\xDD\xA0\x9E\xED{\x8B{\xB4\xA4\xC7R`9O\x08\xDD\x94\x1D\xFAa\xC0\x7F\x8E\xA2\xB2\xDAB\x9F\xD0\x12\xAC\xC95.j|\xD5o\xF4\xEB\xF6o\xE9\xF1$D\xD9Ak\xBD\x09T\x9A!\xAD\xAD\x18\x1F\x0A\xB6\xFD\x13m{i9\x0C\xB8\xFC\xCDs\xAC&\xC5\xAC\x00\x88\x0F\x89\xC9pL\xD3\xB8\xC0\xE4\xF2\x8A"\xD1r\xBES\xB8\x15\xE6\xED1\xACj\xBC\xC6\x816\x8B\xF1\xC1\x1C\x7F\xBC\x13\x93\xCB\xCC\x93\xEB\x8B\x01<\xA2\x9A\xEB\xB7*\xB4\xFB\xC9b\xC9\xD8U\xA4\x01\xC4\x80\xB8*\xBC\x82\xBB\xDC+\x82\xD0\xEB\xD9f\xCE\xD6\xF2\x86\x8Eu\xAD5\x8F\xB0\xBA\x15\x99Y\x0A\xC77`\xB9\x92\xD7$\x0C7\xB4\x81\xB7&H:\xB8oEK
\xEAd\x9Cd\xD9M(\xEBbG\x0Dj\x8Ay;\xA7@\xDF,\xED~\x82\xC3E\xB8,\xD2\x16<\x9A\xBB\xBE\xD1BQ8e\xFCS\x07\x1F\x1F\x8F\x9A\xCCh\x8B\x85\x04T\xC4U\x9B\xC7\x08p\xB0^\xDD&"9\x9B\x9Ek\x89\xE2\x12\xC3\xAC\x12*`\xF0\xC9\x15\xF3XF>+\xF5\x9Eik\xA9\x88\xBF\x1C\xFD\xEB\xF3\xB2 \xD8ld\x13\xB9j.\xFB\x8B\x0A\xE7m\xB6\x97}\x11Pm\xB3\x171\x17\x87c\xAB\x03.\xFE\x10\xDA\xCB\x1DfLg\xC0\xF6\xDC\xA6P\x89\x92\xAEz$\xC4b\x13\xCE\xDB\xF4k\xE3\xAA\xD9\xE7\xBCD\xED\x14wF\xA7\xD2\xBE\x9D\x87\xFC\xEB\xCF\x82\xB3\xF8\x0EZ\x849\xCD\x8D\xCF\xAEd\xB7\x92\x93\x19\xAC\x90m|\x8Cl\xDDE\xFE\x13^\xAB\x91\xEB\x9F\xE5\xC3\xB9l\xF0\x08\x1E\xEBi\xFAG\x1D\xE9\x91\x13\x10\x1B\xBE\xBCP\xDE\xE5\x1A\x14c]\xDAGf\x14\x88M>\x97E\x9B\x1B\xB2\xCC\x86\x1C\xE9R\x0EU\x83\xEF\x9F\x14x\xDA^\xB0k\xEB\x82\xD5\xC6\xBC}26n\xE2\x8A\xAB\x97\xDE\xC9h8+\xC9\xE5l\xBA\xB3\xF7\xB6y\xD3\x7F>t\xB3\x0AJ\x9F\xFF\xFB\xB2\x8B\xC3\xA6:\x82\xEFFN\xB5\xF9,@K\x9C\x9A#\x1BT\xAB\xFEJ\x17:\xA1\xD1\x1B\xC1\xC3z=\x08\x8F\xFC\x08@^\x03\xB6F\x1B@o)\xC0/A\xA9\xA8o\xFD\x1D
K\xB4\xEC\x02F\x9A\x19\xEC\xC85\x03\xE76\xE3*(\x07p\xD0rV\xC6D\x814th\x04\x09\xBB3\x88\x0E\xCA\xD7&\x9EW\xF1\x8D\x1B\x85K_(T\x96\x86H\x87+\xCF\x142\x0B\x0F\xC5\x87\x9E\x83\xDAF\x8C\x84!\x95\xCE\xE6\xB0s\xA4\x0E\x9A3Vn:\xE8\x19\x1E\xFA<\xD1\xD4\x18\x91z\xB7\xB0-r\xAB\xEE\x19\x12\xDF2\x92\x03\x11\xB1\xBF)9\xE5\x9D\xB6\xDB\xE0=\x12^\xE14\xDB\xBC\xC2\x95\xB21\x81\xFE\xEA\xE3\xBE\x86\xBA<\x072\x80\xF8\xEF\xEF5w\xB65\xB2\xD0ot\xA6\xBC\x95K5G#\x15\x0Bg\xEA\xE5O\x97\xD3\x9E\xE9\x8C\x05Y\x8F\xD5\xCC\x1B\x94\x8D\x99\x01\xEF\x8C\xFE\xB2\x14l:\x0E0\xA7v\x8Bt\xCC\xBA\xA2=\xF3\\xB9\x06wy\xE0\xF7\xD8V@\x0E[#\x1Ar\x92\x91\x9F\x0E\xBD\x97B\xE3\x7F\x98\x94hW\xCCH\x0DF\x91\xF2 6)\x14\x07|\x17\xBD\xE5[\xAD-\xEE\xE2%\xEA%\xC0V[`\x1Autj\xF0w\xBF\x8A2*\xA9\xA3\x94X4k\xD7\xE1\xD7\xDB\xF3\x13\xFDa(\x10\xC2H\xF8!\x05\xB5?\x17\x8F\x17\x85\xA7C\xFB3\x87\xFF\x1A\xFA\x16\xF6\xDD\x0A\x935\xCA\xC0\xF3\xDE\x06\xE3\xE3^N06\x9F\xBD`\xB5\xF1\xC7\x1BE\xE3s,\x91]\x08\xBD\xC2\x02\x9F\xFFfd\xCBwh<\x8F\xF6\xBFw\x93\xC5~\xB
B\xC7\xF5\x0B\xC7\xBE\x8Ao\x03\x80\xF7w\xED\xC6*\x00v\xBE\x0F\xE4T\xF3\x0E^\x12\xB50o\xC9\xA6\xF9\xB0\xA0\x04@f\xE4\xEA\xB4\x8F\xDA\x17;\xC8\x1F\x80\xF7\xFE`7\g\xD9\x975\x1Bt\x18\xA3D\x96\x05\x1D\xFA\xC8\xCB\xD6\xF1=7n#\xF5\xE6\xF0$&0\x08\xCDG\xDA\xDB\x0A+)]\xCA\x05n\xEA\xFC"\xE8\xA4\xBCb\xAD\xAA\mki\xD5\xB2\xE2a%y\xBA\xC8\xA1\xA3\x07m\xF5f\x9A\xD4l\x81\xFD\xBC68\x1E~\x88\xD1ThH2\x9C\xD1C\xC60\x8EuR\x0E\x80\x81`:t$\x0BZ\xB8\xD15pZF\xA3\xFB.\xF3+;e\xD1\xEE+q\xA9\xB8];\2[TT`b\xD1e}\xDA\xB8\xE0\x82\xEF\xB6@\xB6T\x9A4\xBA)\x80B&[\x88wc\xE3"7[U\xF1\x90\xF6\x0E6\x88\x85\xAA\xF9\xC6]\x93{\x1C\xC5\x8Auu\x7F\x8C\x9F\x12\x07i\xB5\xD0x\xFF\xFART\x8A\x02\xE7ki&\xBF\xCF\x1D\x8BZ\xD2 6\xB0\xAA\xA8\xC3\x8C\xAA\x94\xB7s\xDCC\x90\xBF\xF9\xD9S\xC2i\xC5\xF3\xE9|\xFF.\xD7k\x85@W}\x18R&A\x8F\xE7\x81#+\x82\x0E\x99\xC9\xDB\xF5\xBF\xFB\x82{)\x84\xA9\x98\xDA]\x19\x1EL\xF8\xE4\xED\x0F\x15\xB58\xEB1\x1F\x89\x05\xB3R\x01T\x98n\xDB\xF6\x14\x17\x13\xE6z#\xC3\xA9z)\xD2\xCC\xAB\x10}\xCFy\x18'\x0F\xE3yX\x1D\x9C\xAF
\xDB\xC6n5\xB6\x9C\x13\x8E\xDD8);\xBB\x01\xF0\xAD\x82&\xDC 1\x16qz\xE4\x95\xB0\x9B\xA3o\xC5\x9B\x1B\xCE\x1C\xF1\xBE\x05\x8AG {~\xF0qp\x06m\x0B\xFBD\x14\xB7\xD2:@ii\xD0\xDFD\xB1\xAF\xB9\xA4\xEE\x03\x92\x08Ze\xED>L\x8E\xF6\x1Ev\x8F\xB8\xBB\xCE)N$\x04\xCB\xE3p\xFE\x1C\xAC\x1A\xCF\x7F\xB2\xC4\xCC'S:\x15\xF4\xCC\x7F\x0FUr\xD4\x08\x902\xC9\x88\xEE\x81\xB3B\xB8F+\xC8\xA1\x07\xE8e\xD1\x8A\x9B\xB2\xCE\xF6>\x86<\xDA\x16l\x8C\x1D\xED\x0E\xBB\x8Ce\xF4?I\xFD5z\x04\x93\xC17\xC6\x97HY\xCF)\x0F\xB2\xC65Q\xA3W\xA4\xCE\x8F?\x17\xE6\xBB\xCC\xADD\xCC&\xE1\x1C\xF8]+\xE51\xDB\x19\xFEW\xC3\xD9\xF4\xBE9$\xEE\xE1[9\xE3P.\xDB\xDEP\xDF0\x8A\x8B\xF7iHg\x98\xE1\xB6\x19\xA5\xB2@\xDC\xFA\x13\x98fa!(M\xB0A\x1F\I\xE0\xBD5\x84\xFA@\x98\x14\x97\x9Ey\xE9\xAE\xB8{D\xFC\x7F\xD5\x0AS\xD5~\x80\xF6\x9Fk\xC1g8\xED\xA0\xDF;\xDE\xDC\xE6\xC1\x97\xA7\x19\x18\xDCJa\xF6\xC3O\x0A]\xF685\xA8\xB2mzF\xDC\x9B\xA4\xAB\x13\xBB'\x84\x00(\x82\xCD\xB2\xF3\xC3D\x05\xCF\xA2{*v\x180gU>\xB4\x82\xA3?\xE5\x8C/36\xE7(\xD1\xB9\xEFs\xDF\x09\x1E\x95
\x9E\xC3SQ\xF0\xDA\xAF\xE3\xEC\x82H(W\xC0\xD5\x1F\x87\x98@S\x0B\\xCC\xDDGn\xC6Q\xA0}2\xF9U\xCB|\x9B\xB1\xA7)x\x9F\x93\x05\x1D\x9B5\x19u\xCCNfB\xDE\xE4\xC8z\x99-a\xD3\xDE\x14\x16\x91\xF8\xA6JZ\x0E\xFC^#\xC7\xD5v\x15\xB0\xA1\xF1\xAE\xA9J?\x8FKZ\xE8\xFF\x0C\xF2(^\xCA\x1F\xF4\x80\xCC1\xAB{l\xF54\xF8&o\xD4\x8E\xB76{\xECC\xA6U\xC3*\x8D\xB0\x95\xE4\xAE\xAD\x1C\x8F\xD2\xC5\xCDY^\x0E\xD0\xBD4\xEB\xC8\xD4\xBCx\x82\xE3K\x1A\xD2\xEB\xCBB(S\x0F\xA6`\xCE\xC5\x10\x19\x8F\xC6\xAAK\xA4\xB2\x14\x09\xE4]h^{\xC9r\xAC\xDC\xDA\xDB4\x1F\xFB\xC5r@\xB9\xE0\x90u\x8F?J\x0Bv\xD1\xCE\xF6\xC7\x8Do{\x9B\x96\x9C\xCA\xDA\xE8!s\x01f\xA1\xA3\xC8*JP$\xB0\x1F\xD9\xDA\xFD\x1D\x1A\xC2\xEF`\x8F\xCC\xC6\xEAM\x1A\xD0%`\xE5Ja\x85\x81x\xA0\x03\x99\xFF\xE5\x190P\xED\x89f\xC0I0nK}\xF2\x00)3\x89\x8B[Os\x15UF\xA5B\xFD\xDAR,\x1C<\x8E\x81\xAD0\xB6hW\xEC\x8F\xA4R\x95\xA9:(J\x94\xA3\xF8xWH\xE2\x8B\xC5\xCD\xA8\x7F\xBF\x0E\xF8x$\xFD9k\xF4\x08iJ\xA9n\xC4_\xB7d\x81\x91\x09\x9B|\x19x\x92\x8E\xC5%\xD0*\x8C!%Q\xBCSO\xD1R\x14\xE3\xBB\xAF\xD7
=i\xB4\x1A\xFB\xC8\x1B\xEC\xB1~\x12\xCA\xB8)\xFB\xC5\x9E\xEE\xDBj'"T\x00g\xADc\xD8\x89Q?\xECe[\xF6N\x91I\x10\xA8\xA6\x14\x1D\xCE\xB6\xF9O\xE6\xACk\x1E\x95 \xE9\xEFO\xF8j\x0C_\x00Q\xBC\xA7\x9E\xDFN\xD7L2<\xCF?\x9Fa(:\xCD_z\x15\x06\x17m\xF0=\x86\x8D\xF9\x1D\xA6.?\xC1\x95\x9B\x85\xCD:\x88d\x15+\xCC\xC5\xA6\x8C\x81&\xDE;\x03\x17\x7F\xC8d\xCB\xB8\xE6b%v\xAAK\xB4\x13\xBF\xE9\x18\x88,q\xA8\xB6`\xF7\x84\xC7\xF2\xDD\x11\x841\xF6\x94\xF5mh\xB7(2\xB3{\xF3\xD9|\xBF~\x1F\x02>\x86D\xFE\xFA)\x8E\x93O\xAE\x95\xD7\xC7\xB6\xC4\xB1\x92vc\x8C\x11\x1A\xB3\xCE\xBE*D\xE54(\xA2\xCB=kb(\xCEA\xDC6\xFA$\x1C0\xC5`\x88\x05\xBC\x1D`L\x08\xAD\xD0\x8D\xF5\x19dh\xDD\x1F1\x9EB\xAF\x10P\xAFM\xA5\x10\xDF.\x0F\x00\xBE\x81\xBE|\xDE\xC5\xA3\xFE!\xBC\x15\x1E\xEC\x8C\xC2\xE0\xF6)%\x12\xDD\xD7\x0C\xAE\xBB(8LZ5\xEE`\x18!\xEA\xEElI\xFA\x0F\xB1R\x86\xF1\xB2\x91\xD0\x0D\xAFY\xF4\xF0\x82@n\x1A"\x98~\x8AMD\xEC\xCFp\x10\xBDT\xFCR\xB6\x97fs\xBDM\x19\xFD\xEBA\xAE{)\x82\x08s\xADI\xAA\xF9\xA0\x0D\x9DVh\xAF\x12~\x0B\x83\xC7\xBD\x05\xD5
\xBF.\x8Dm:\xFF\xBFE\x8AP\xEE\x06.\x10\x0E\x85.B\x01\xBB\xB3\xB2r=\x81W\xAF8,\x86A1\x8F\x09\x1B\x05\xEE\xC7s\x9AQ\x97\xB2u\x11H-R\x1AL'\x89\xFBV\xF1\x0D\xF5{\x85\xDF.J\x1Bu~\xB3\xB2=U;Y@\x03\xE8\xB9n\xB8\xE2\xB6s\xF1uxHhv5\x97\xD8\xBE\xEE{\xF1Y\xCA\xF8\x88\x0D\xC9\xEBS\xB6\x0Dl\x14^\x9A\xF1\x89\xD8\x82+\xC8\x90\xD4\xF6\xEE\x04\x14\xA8H\xBA~\x89@X\x9A\xB9|\x80\xE9\xD6\x0E\xE3> \xEE\x12\xF6\x92\xD0z\x9D\xAC\xC8\x1Er\xF7\x07T\xD3\xAE\x99q\xC2\xEE\x0Ak}i~\xA3\x87hr\x9D\x85vmh\xF9\xD0?,\xFC9\xAF\xE1\x93\x99/\xBF\x7F\x90\xD0\xC3H\xC5\xE1\xFB\x83\xBF\x852\xEF\xFB\xAD\xE7\x08~a#h\xACh\xA9\xE0\x84P\xB9\xF1Qhhyp(\xAEVnz\x08\\x8A\xDDT\xCCV\xB3\xEE\xDA\xFA\x0FVC\xDBE\xE1,|\x0E\xD2\xEE\xEB\xB7\x04M0\x91^\xF1O/\x03:H\xA73\x06\x87\xBB\x1EB\xFD\xB0\x1AR\xC1\xB1])\xE4e#\xF7\xC0\xA5\xB8)\xA0,\x16(\xA1\x80C\xCDQT\xE0\xE5\x7F\xBFG`<\xF1\x1D\x17\xDA\xED;\x80\xBCb>\x9E\xB5\xBE\x04\x86P\xD0\xDB\xC7\xDF\x8D\x06"\xF8.\x8B\xC3U\x00\xD2\xE9[r\xF9\xFCf\x1C\xB5n\xB0%lYU\x15\xC2\xC5\xB5\x89\xFE\xE2\xE4\x9E\xFF\x
92O\xD0\x09\xB4\xE4\x81\x02\xDF\xC8G^\xC3\x03\xC66o\x8Bf\x1C\xEF\xE6\xBF.\xF3\xD6\x13h\xD6\x10\x9Cx,\xC4\xA8\xAD\xF0\x03\x87Oy\xA5z\x0Fw:Y\x0B\xDE\xF2\xEE\xA3\xF5\xBCS\x0A\xDD\x10c\xBF\xB6\xB2\x95\x02#\xFE|j\xF3$\x1F\xC5\xBEP\x86\xAC~O\xA3\x92Zs\x19K\xFBVq\xC2+\x8F\x1F[O\xA2;\xB6?\x0F\xE2L\x81b\x98ymr\xDEK\x86EZ\x0A\xE7F[\xAC1!7\xAF?\xFD!\x04\x1D\xFFe\x04d\\xC6@\x1D\x91\x05\x121\x18d\xE5S\x88\x94\xE6\x08\x17\xB7b\xB6\xC1\xCD_\xEA\x98\xF7\x87\xEE)\x8D\x16\x878\xE8'\x9D\xD7\xBCVT]\xF2\x8C\xCE\xC4O\xBAkdI\x08\xD3\xEB\xDB\xF6TAf\x1B\x15ZQ\xBF\xCE\xB0[@\xDB\xB3\xD9\xD7`J\xD5|\xC23\xEA\xAF\xC8+\xBE\x84\xCE\xA2\x0E\x97\xA5T\xE3\x8CN\xA2\xF3\xFA\xA5\x9B\x08\xDB\xBD\x14\xBC\xA1^\x16L\xF0+\x04\xAA\x91\x14\xD6\xD7\xAB)\xAF\x97\xDF5/\x18G\xD3\xD4\xD9\xFF\xF2\x96)b\x0F`\x86\x01\xA6\xB7\x0D\xCE\x12\xA17b6\x80\xEA+a\xCA\xABxl\xE7e#2y\x0D\xDF\xCEC\xCE\xAF\xE0\xCF\xA1Yu\xBE\xC1"AF\x1C\x85\xC8,a\xCC\x8E\xAE\x04\xB0\x8Bq]\x0C\xE4\xF1\xBD\xB4`\xB8[\xB4o\x905\x06\xBD\xC7\x84k^\xB6Jw\xA6\x808\xFC\x83\xC8
^%\x07\xAFEp\xC1\x02\xBCh\xBE\xD2\x08m\xCCl\xB3Q\xD3rf\xB09\xEC\x95\x17IA\x83\xB07\xC4\xFB\x862\x0D\x98\x83\xC3$\x00\xFF\xF4\xD1`*\xB6\xA5Ny\x02\xA9\xECnLP5\x7F3\xEA\xE6\x92\xA1\x8C ^[?\x85S\xBBv"x\x9A\\xC2&k\xC3}\xC1"2\x8FT\xC0RGj\xF2T\x1B\x80\\xA1\x96\xEB\x9FPo\xA8\xE8\xA5\xE1A\x8B\xE6\xE3ld\xD0{\x9A\xB2\x05;\xC9\x81IJ0p\x9DpV\xAA\xD5)\x14\x15\xC2l\xB8\x9A\xDC\x9F7\xB59J\x0E8p\xC7\xD4\x1E\x84\xD2I)\x19\xA3L\x9F"F\x14\xB3\xA5,]\x0D\xAA\xC7\xA9D\xA0\xE22\x12\xC0\xD3\xA9\x1B\xB3\xA5\xE5,\xC8\xA7\x92\xC5~\x19w\xBD\xC1\x93F\x19P\xF4\xF8\x92\xD1\x7F\xE5\xAF\x8B\xD4+4\x0F-B\xC1[\x9A;\xB9\x8C\x04\x9A\xD0\xC7YL\xB10a\xC7=\xCBX\xE3f\x9D\xC596\xFD\xC4\xE9\x04 \d\x88\x8A\xF3\xA2;\xB7\xA4Z\xD9ds\x86D\xAAM\xCC\xCC~8\x0E\x17(\x82\xCC-$\xA3C\x9B\x8F7\x9E=f?\xBA\xC3s9n\xAC\x0A[\xC5<\xF5\x08Enc\xF3\x9D\xFDR\xCF\xDD\xF0S\x88\xF0G@\xEBD\xE2g\xD1v_\xF3\xE7w\x05\x05i\xE7\x94-\xDA\x89\xB3\xCB\xEF5;\xBCzZ+&-\xE8;\xB4v\x8B6d?\xB9\x1DJ\xC7^\x13\xF5\xD9\xF1X\x9D\x1AZ\x9C\xA2\x14U\xFEw\x88>X\xC1oSj\x85-\xE9\
xFFJ\xE5\xF1\x1E\x9EeXYuL\x89\xE6`\xB7*M]Q\xC2\xA9#\xAD\xC7\x98\x8B\xD1\x0C\x88p6\xD0E\xD9c\xD5e\xD5C\xC3\xB6\x03E\xFA\x93\x88\xB2\xC4\xD06\x16\x09\xACi\xD6wO\xAD\xFA\xB0J\x83\x0F\xB4\xFE(\x1B!Z4T\xD1\x9F!\xAD\xD1\xFF\xE7\xA13\xFF\x8D\xD9\x05\x1AU\x9F\xE7(\x0A2?I \x05`\xE7\x1D\xB4o/\xEB\x81\x02\xD1\xD1P\x05\x8B\xC9.\x04\xE4f\x91\x82^\xE4\xDDh|\xC2\xE9I \xB5\xBE\x9D~,\x9A\x1D\xA2d(o\x07z^\xAC\xBFJ\x96\xD2=">\xE3\x90"\x16y.\xE21\xB7\xBB1\xB0+_\xDD{=\xBCD\x97\xE2\x87\xB6\xEBG\xA6<+)\xF1v_\xE2D\xAF\x04\x04\x7F\x9D?\x11\xE8\x89\xA7\x03r\xBC\xCB\x9C\x94j\x95\xC5~\xDCo\x9BK\x01\xD3\xAE\x88\x9Az]\xE5\x04\x17vm_\xEEf\x8D\xCCC\xF1\xB2\x18A\xD89\xF4\xDA\xD8\x82\xBFF\x15\xF9\x05\x86\xA5\x1CO\x981\xEC\xFD\xEF\x9E\x9C\xF8;\xBC5\xA6wx\x81\x16X\x80;8\x93\xBF\xC4\xCD0\xC7\xE1'\xD3\xB8;>\xC5'\x89\x11\xE5c\xE0p\xCA\x0A\xC1v\x0C\x0C\xB8\x1B\xCFM\x83\xE4\xDFY\xB5\xBD\xD3\xEEE\x87\\x1C\xAF\xA1\x94\xF1\x89\x80\x83B\xDFX`\xD2h|\x1C\x94\x03\x8A)fp\xF6T\xEF,\xD74\x0EO\x0BU\xFA3\xBC\x9A5\x953\x07\xA3q\xFCf9u\
xF8\x97D5\x11.\xB0f\x0Cz)]YB\x94&\x1F\x092\x84\x0D\xEF\xAB\x0A\\xE5\xFA\xA8\x81m\xBDE\x9C\xC1\xC3\x88\xA0h8D\xA5N\xDE\x17\x9EK\x15\x09\xFFUfH~\xB0J\x1A\x01B\xBF\xA8\xD3\xA4s\x8F\xFF\xB2\xA0\x18#\xA7\xA5n>\xC1\xAA4b\x85\x13XB+\\xCD\xF9\x7F\x82\xE4\x82y'\xBB3d\xD9\xF2Z\xD4\x19\xFB\xC0~\xBDj\xE57]\x1F\x0A\x87W\xEC-\x11\xA9\x96\x0C\x01\x8B\xB5\xC5\x9A\xBFKn\xA54KDi3\xC1S\xC3\xF8/\xBE\x090\xD9\xD0\xCD\xD8*\xC0\x9F\xB1((_\xBF\xAD'\xE6\xCA\xB1\xFC\x86\x84Y\xB4\x03\x1ER\x1A\x13(\xAA3\x03\xD4\xF6\x8E76\x9EV\x05\xF8\xA3\x1D\xC6\x0Ai\x9BUv\xF4\xCF\xC9\x01\xAD/\x0A\xED\}K(\x1B\x04\xE6Je.\x8A\xBE\x16C\x82\xBA\x0B:\xE7dwB\x10\xDB\xF2\x1BT/\xEB\x86:\x9D\xDA\xE2\x10\xA7N\x1F\xE7\x91\xFF\xEE\xED\x08\x93\xA4\xEE\xE1Q0w^\xB8\xAA\xF7\xD0\x99=\xB1z\xD9D"kZ$H\xF7\xD1X.\x1C?Wn\x1Ds^1@\x82\xA6\xB0\xBD\xEBa\x1B\x91\x11\xC2\x8El\xCA\x8A\xA5y\x129\x9F\xF5\xA1DY\x1D\x91\x0B{-\xB8\x16\xC9\x1E\xA7\xCC\xCF*\x09\xEBE(\x0D\xA8>iP\xCE}\xC8\xEE\x09\x04\xFD\x001\xF6\xA3\x1A\xCEi\xDCR\xBAI\x1DA\xDD\x94\xEC0An\xB9\xA5/7
\x01X\xA5\xB3z\x183)\xEBf\x14\x0C\x84\xF5\x97\xF1\x17\xC2$\xD4_l\xC8\xC5\xB4G\xFCE\x90k\xF0\x92\xD1\x0B\x90(m\xAF\xAB\x065\xBDr\x93\x110\x14\xAE\x80\x84.\xA6\x0D\x0D\x94\xAA|\x8F\xDB\xAC\x03\x97x\x8F\xF4#z\x10\x1D\xB0\xE9V\x0BY\xB1\xF7\xAA\x19\xC4\xA0b\x1E\x94\x8B\x1F\x9D,\x92q\xCF\xF2;(\x83k\x0E\x02\x83\x7Fr\xDE?\x1D\x18\x8Fe\x0D\x99\x13\xA6\xBB\xAB=\xB6\xDDf~u{N\xF1\xDC\xF6\xBCR\xDC\xF0\x16\xA0BZ\xA9b\xD9\x8B\x0F\x16\xB0lN\x9E\xF13\x19\xDE\x89\xAD\xACq\xAC\x98\xB1\x96\x97.\xADj,IQ\xA3\x91\xC2\xE6\x8E\x13\xE9C\x7F\x08\x8D\xDE'0)\x14\x05\xBE\xA1\xE6]\xE3\xD7\x16V\xAA]\xC1\x0C\xAF\x87\x8E\xBF\x05\xF3\xAB\xA2\xFC\xC7~\x97\xA7\xAA\x86\xE1\x90\xD8-\xA1JD3\xD3\x9D@\xD9\xA9\xA3\x96_}\x97\xC4?\xC8BvQ\xAEC\xE5\xD0\x8DM\x92w\xEB\xE09\xD8\x96\x05Z\xC3\xEB\xFB\xBB\xDC\xCE6B!k2\x96\xF8\x80\x19]\x83Y\xAA\x17\x0Cx\x0A8T\xBF\xF6\x7F\xED\x18\x0E@]\xFA\xF1\xE2\x04\x11\xB8\xD1W\x81\x8DB\xF6\xFD\x7F\xFE\xC1\xF9x)H\xE7\xE2W\xB8j0B)\x91\x1A\xED\xDA\x88W\xE1\xBA'&\x83\x82\xCA\x8E\xE6\x96\x94\x05m\xDE\xF8
\xEE\xBAa\x8C\xC2Y\xAE\x99\x8E0\xAB\x18k\x9A\xA7\xA6gSk\xA2RU\xD9\xE2\xA5\xC2\xD9[\xA4\xDF\x83\x8B\xE9\x92u\xECn\xAE7q6\x1Bw\x80b\x0EG\x87\x8C\\x08e\xFB\xCA\xC7\xFA\x80\xF8\xE7<\x12f\xAA\xB7:pg%l\x1F\xC89d\xA5\xF6\xE9\x07\x01\xCFp\x90w\x874\xE7ck\xA9oH\x88\x93\xF3b#\xABj\x83^\xC2\x05\x18\xC9\xD9\x183t\xEF\xF2\x18i\xFA\x8D\xA9`SW\xC3\x16\x8F]\xC8\xAC^\x91p\x12b\xDD5F\xA0\xED\xC1\x17 \x0C\x80\xA1\d\x8D\xD1\x9A9\xD2\x10.`S\xD1\xC7\x8E!\xE5\x00\xF5\x86o\x9D\x96\xC8M\xE2"\xEC\x82\xDE\xAD\xD5\x11w\x8D\x90\xDB\x86^\xCD\x9A\x17\xFB\xD4\xD4-\x88\x9D\xE8\x90\x8Bt\x84)\x16\xA2g\xF6\x12\x17P\xD4?\x05\xBC\xD6GC\xEB\xDF\xA8\xFA\x00\x16\xE3\xB4\x10l\xCD-\x10\xBA\xB2\xFCst\x06\xD8\xCB\xA9&\xAD\xB8hR"~\xCE\xBF\x19{2\x17\x8A\xCE\xBE\x8E-\xA6\xE0^\xA5\xDB,\xD19\xAEt\xB7\x9EtM-ch5.y\x98d\xD3F\x99D\xAD\x9C\x90#\xD96nO!\x8Ek\x9F\xDC2}\xDF\x9F\xCD\xA2\xEAW\xF8\x00a\xF4c-\xDB'\xFAf4B\xE2X\xCE+\x04qd\x99_\x05^o\xEE|\xAF^\x95V\x87\xB6\x11\x95\xC9\x7Fp\xC6\xC87\xFB\x98\xE6\x83\xDE9\xCE;Zo\x06\xA5_\x95k\xCF\xA
F\x19\xFB\xF6\xD8\xD9zi\xB9/\xFFP\xC0\xCF\xC0\xDBq\xF8\xC4\x89h\xCE\xD3\xABUZy\x1C!)2\x89t\x1A\x1B@Y5\xD2\xF8\x1DJ\x0E\x0F\xAB\x17\x8C\x0F{\xE5\xCC\x90I\xB4]%T\xD9P\x85gr\xC3\x7F;\x15!\xC4\x06\xBA\xC4\x83\x8AY\x1AgsT\xF1\xF5~\xD0\xED\xBA\x18\x0B\xF5\xBA\x91\xC62A2F\xA9\x94\x18\xF7\x9D\x86[{8\xBA4>\xC8TM\x9D\x0D]\x1B\xBE\xB7\xEB\x92\xF9y\xA5\xB4\xB1i\xE2\xD4 FI\x9A~\xB8m\x93\xCFV\xD2i\xD4x\x1D\x0C\xEE<E\xDC\xC9Y\x1C\xB7'\x80\xAE\xF1ZG\x90M\x9E\xC5\xFB\x90\xBF\x93G\xF7\xD00\xEC\xD7fS\x88\x8D\xA8\x92,\xEA\x9F\xF0\xC0\xE0B\xBB\xD9\xAB:W\xA5}\x02\x9D\xA1\xD9\xA0\x05\xA63\xDFs\x95-4 \x1D_\x0E\xE2 L\x01%\x19\xF3{{\x1Fr\xFB`\xF1\x83\xB2\x9B!h_\x9D\x0A\xE2\x1E._\x0D\xF5\xF2s)\xC8V\xDAYF\x1A\x86i\x08:J\x06\xFC:I;\xB9\x8B\xAFo\x09\xFA\x1C\x00\xF4\xA2\xB7_P\xA5\xDF$d\x9D`\x80\xD5\xAC?\xCA\xA15I\xA8\xCElN\xCBCb\xCB1\x04q\xE5\x0D\x8D/zab!\xFD9\xD0\x9B\xFC\x89\xAE\x00\x03\x8A\x03\x1F\x8A9\xB9\xA1\x98L%\x00\xC6\xAE\x86\x04F\xF1X?\xE7i;|\xF1w\xE2\xB9x0\xE2_\x94\xE8p\xD5+\xEC8\xEB\xAA\xBCR\x82\xED,\x
1B\xAA+\x8A\x87P\xF6F`\xE0`9\xD0\xB9\xF1\xDE\x16i.\xAF-3\x1C\x1Ax\xD3^\xE1g"\xC3M#\xE11\x18\x92\x084;KHh\xA8\xB8 \xBA.\x05\x8E8\x03\x03\xEAr\x90\xDE?\xAFF \xA7\xDC2Q\x99\xB2/\xB4X\x9E\x1C\xE6\x08$Q\x19/\x12\xD9\xFB\xE1\xDE\xB0\xBFo\x9B\x13~#\x0D5\xE0dS\x06-i|\x01l\xB20\xD0\x14,\x8D\x19Xi\xB6\x8DZ8ej\xB3\xFA\x1F!\xA4\x01\xA8Y.\xCE\xE2\x14\x93\x14\xA9%\xFEK\x12b)\x9B\xB7D=\xB0\x86\\x19\x93Oz2\x8E'\xFCO@T\xDD2\x94\x18\x8A2cN\xEF\xF0t\x98Fl\x19\xBB\x1Es\xE9\xC7&@\xE1\xB9O\x17\x17\xF9\xE1\xB1J\xF9\xD1<W8\xC9\x0EG\x03\x8C\x8D\xFA\x8E\xE4\x9C\x0E\xD1\x8D\x0B8\xC2\xDAm\x0Dy\x1D?\x98\x15\xDC\xB0\x83\xB11>\xE7\xB4O\xFFRs\xD7e\x9Cxeh!q\x0C\xF9\xC4\xBD\x12R\\xDF}\x0Ag\xBB\x05h\x08\x06&M\x14a\x080\x81z\xD8\xCB\x90\xB9\xEC\xF6\xA1v\x7F\x96\x17\xA0\x0E4\x1B>\xAE\xDEw\xE4B\xA7B\xF0\xF0$\x0F\xBA\xF8\xA0I\x83RV\x19\xDE\xC2]\xC4\x98\xF2\x1C\xFE\xAB\xE5n\x9E~~\xA4\xBA\x11WpK\x8B\x9C\xA0Y\xED;\xA4\x08!\xB3`'o\xB6*\xAB\xBF\x84-\x06&<\xEC\x8Fl\x82I\x87u\x02y\x1D\x9F\x89EG|Qc\xB5 \xE9\x8A\x10\x07\xF0\xDFA\
x00\xB5M\x85\xDF\xBE\xB1\x8D\x8Bk?\x1AOL\xAC\x97o\x10\xB7\x0D\x9F\xDFY\xE8\x89_6S\x98\xBBo\xE7H\xD0c1\xEA$Q\x18\x92\xB3\xF9 \x89i\xAAOy\xAABe\xC9+\xD41Jl\x0Ck\x18U\xC4\xE8\x08w<I\xF6}w\xD1\xE3\xE8rV\xED\xBEC4\x06\x87S\xB9\xF2\xB5z\xC0\x84\xFFv\xB5\x00\x9B:\xCB\\x11\x08M\x1D"`\x0A\xA5\xFF\xA7/\x7FV\xFF\xFF\xC3\xFF\xFD\xE8\x83\x95\xC2W\xC0E#fW\xA5!g\x89\x8F\xDBF\x1B\xAE_\xA2A\x03\x18\x98\xCF\xBF\x98{\xC6\x1Du\x97\x0Cqv\x96m<\x88\x8B2\xA4>\xD2\xCF\x03-\xA38\x15W5\x81eK\xBE[\xBB~\x90\x9FB\x96\x0C",7\x8D\x7F\xEDZ#\xE7\x14J\x1A\xE4y[\x99f\xF8D\x9A\x82\xF1I\xF0HAR+w\x14^\xA5\x09\xB1\xF9\xA4\x10$\x15?VQr\xCF\x90\xEF$$P \xB2\xB18\x1A\x1F\x88\x0Flk\xEEL\xFFo\xD4\x10\x8E\x82\xD5M\xF1x8\x0A\xB9\xD5^Qy,\x8Aa.D\x84B\xF5_\xE8\xE6\x8FT\xB4\xCD\xF8\x8F\xB2\xB4x\xF0+"F\x07\xF6\x158\x09=\xEDl|v\x08W/\xB18\xB4\x9C\x88\x04PL\xE2f\xB7(\x1F\xFB*0\xA1\;\x88?\xE8\xBB\x97\xE0]\xE3\x00\xA1\xAE=\xBC,\xCA\xD7\xF1\x11\x00\xD3\xC4\x8A\x818\xC9p\x9Fy\x87@J_\x86"\x85:{\x8C\x89\x02\x08\x071\xD9\xC9NB\xA9{\x8B\xEEO\x
E5_\x0FE\xA3\xB5\xF4\xF3\x0D\x9E\xFE\x84\xD1f\xD7L\xA5\xEA\x9CF\xC7\xB3\xD5\x9DI1R\xC0\xDA\x96\x06\x0E\x07v\xD3\x9E\x0C\x84\x9F\xD8-\x9D\xC4\x1E\xFC\x12\x0C\x01zv\x0Dr\xD7\x18\x02\x08\xBF^7|\xB5\xBC\xDF\x98;\xD6\x10\xA4\x00[\x89\xAC'\x87q"\xDE\x00*vS\xB7\x8E\xE2\xB4\xF63\x09\xC7\xD0\x99`3\xC9\x9E[\xC3\xE0\x88\x82\x9B\x92\xAFK\x8Eq\x94\xD3\x12\x82W\x06\xFCJ\x84\xC8<\xD5\xFE\xC6\x8E>\xBB7\x828\xA1\x0D\x94\x7F\xF3\x85\xA0\xDC\xAF\xDD\xCB\xF4\xBB\x94w\x14\x80A7\xC0\xBA%5\x0C\x10\x92U\xBB\xD1\xB4S\xB3y[\xDB([\xA9\xF5T4\xB2\x982L\x0E\xB0&L\xBE7Aq\x87\xC7[{\xF9\xF1\xA4\xA7\x12\\x9BO\xFFu\xB1\x06V%"\xA1\xAB\x85,\x9F\x8E\xA4\xEC\x9C\xD3%\x0E\xFE&\xFE\x14\xD2dN\xE2\xDA\xD3\xD5\xA7\xF3\x0Bu\xEF\xDA%\xDC\xA5\xA5\x88vji\x84\xFFC\xFE\xA0\x9F\xC7\x97{I\xD2\xC6\xAD\xE7qU[\xFC\x91;\x86\xA2,^\x92o=\x93\x0F"\xF1\x9CY\xB0\xA8\xC0h4\x89\x0DO\xFE\xBF_\xC8PH\xD9\x8C\x91\xD1v\x88\xF6h\xC7\xB4R\x92\x9B\xD8\x96\x88D\xC2i9\xAD"\xE9\xD3w\x16\xD4t\xF9\xA6\xAD~\xB9\x06Zi\x0E\x14 m\xC3\x9E\x9FzX6\xB1R\x11\xC8"\xC
C\x85Z\xB7H,\xA7gZ\x9C-8\xD1\x1A\xCF\xA0\xC3\xA1e\x03\xAD\xD9^\x96i?\x86\xA6\xB1\x94x\x97\x12\xC3\xE1\x0A3~_\xB7\xA5f6\xBEmU\x80\xB0\x9E2\x185\xFA\x9Av\x17p\xE2\x04|~`!\xC5\xCEB\x98C\xE3\x15\x82\xA9\xDF\x0C_#\x12R\x19\\x092\xA4\x81\x06\x04d*\xCD \x1C\x91\x8F\x87rq\x17\xD8\x937\xD9\xE8\xD1\x84\xFD\xB2\x95\xD9z\xFC\x97xo\x80\xCFe\xB6\xBD\xE2w\x0E\xA9\x03\x13\x8F8\xB5\xA1\xAC\xEF:\xAF\xBE\x1Ctv\x09\xA0!m!%m]\xB6\xB2s\x95C%\xB5\xCE\xC0nv\x86!m\x81 \\xBAz\xE61_\xBC\x0D[\x8E2\xFBJl\xAAhi.\x98v/F\x18\xA1\xB0\xF51A\\xE7\x02\xFEu\x8F\x1D\x0E`|\xCF\xBF_\x9A\xD7\xB4f\x10\x84|\xF6\xEE=\xA3\x050\x81\x85\xBA\x94\xEE\xF0\x8B\xF7\xEA\x0EQyj\xCBo>\xF7b\xF3\x11\x9D_\xBC\xB9a|\xFA\x05cl\xEEQ\xE5\xB5+\x12\x82\x13K\xC4\xA1g\xDF\x1D\xA3\xB6\xBB\xDF\x19\xB4H\xA6\xF6\xD6j\x9DB\xD2\x9D(\xB4R\x11\xC6\x9A\x8F\xB2;\xE8~\xE8\x0E'\xE4Ru\xA9\xDB\xDC\x99\x06\xD4a\xC2\x09\x84\xB6\x91\xAA\xEB\x00\xDC\x1A\x8D\xC7\xB6\xD0\x14\xDF[\xFF\xD7\x1D\xAEW\xA4\xD1\x00\x13;\x85\x1B0\xC7\x18\x08b\xEE\xA7\xED\xA6\xB9\xB8\x12\xD7\
x86\x9EG\xBBl3\xAF\xE0\x1C\x169\xFA\x8A:s3K*\xEF\xFC\xF4\xB75\x98'\xFAM\xBB0[M\x1B]*\xAC\xD1`\xC4:\xDF\x1A\xC4\x9C\x0A\xBE\xF9\xFD4s\x7F\x1C\x0D\xA8"\xF6\xF9ZY\x99\xAF\xC9\x1B\x902d\xA8V\xE8\xDFi\xA6a\xF6\x88\x959*\xED\xB3(\x91\xC0\x8E\xB3\x93\x8D]\xCFC\xF1\xDDI\x01x\xBF\x0F\xA1qh\x84y\x7F\xFC\xFD\xA8*Gz\xFDX\xB4\x15s\xCC\x09iI\xC7\x14\xF1\xB6K\xE2\xF7\xDB\xF35\xC6P=\xCE\x80\xE74qx{\x1Am\xC5,\xEC\x0Fr`\x0F\xB8-\xC3\x10u\x00\x17\x7F\x83MZ\xE6\xA4\xD3_3\xB2\x00\x81n\x93\x88\x8F\x0B\xDD\xEE\xEC\xD4\x99\x1C\x90=\x9B\xA5\xDB\x85\x16\xB9\xE1\xAC:\x06\x08V\x92S\xF1\xCE-\xB6\xC65\x89\x92W\xD9U\x879\xE5\x16\xE3\xAF\xDDi;\xD3\x15\x93\x81 \xD0c\xE9r\xE5\x05-\xDC\x90e\xB7\xD8\xB99\xD4\x1C\xBE\xD4IS\xC7|\xDC^\xA3\xA3\xBC\xCF\x0B\x80\x97\xF6\xDB\x1F\x8A\xAA\xCF6\x0B*\x82xw\xFD\xDF\x86\xE0w\xD9\xEA\x80<B3c\xB2\x99\xE6\xB0V'\xE0\xF0\x19*\xEC\\xC4\xF4Lz\x9C\x86\xED\xFB\x1Fd\xF8\xA4\xDB\x06\xCD%;\x8E\x89G\xB3\x1D\xCF%\xD5D\x903S9I\x06Y\xF6\x86\xD1"\x7F\x88\x12M\x89z\xA0\x09\x04\x188[\xB9\xD1\xE6\xDF\
xA8\xEC\xC7\x1C\x96p\xF1\x01\xBC\x0A\x17\x07dW\xACd<\xB1\x1E8Z\x12d\x99\xE7\x82\xF2\xEC\x1B\xD4/\xA4k\x95\xE8\xDFjL\xAE\x95\xF6*\xA4\xEE\xF7J\x94<A\xD6\x10\xB7y\xA9\xF0x\xE7\xF9\x0D\x01\xDD\x8CA\xE9\xB7\xA2\x7F6\x89\xC2\x12\x15n;I4\xE3\xC9%:Uu=C\x83S\x07do\xDB9\xF1\x86e=\x86;`O\x1D\x16\xB0\x15\x99\xE6\xCC-\x83*\x06j\x1F\xE7\xB3\x14{\xFB\xAE\x0Br\x87QM\xD7\xAB\\x1A\xC7\x0A8\x10\xF5\x0D\xE0\xD3\xD0\xB1h\x97\x86\x12\xA9Xb?\xB1.\xA2\xD0C\xAA\x89\x8A\x01{\x12\x9A\xCD\xC2\xEDz\xE5\xF0\x01M\x01D|M\xF5jo\x9C\x11t\xAE\x82\xFC\xBB\x1Ck\x01\x0F\xE5,\x8C\xFD\xC5 \xCA\x8A\xAD\xCC8\xC7\xEA'%\xF3\xF1Q\xA0\xC1t\x126;b\x84\xD8,\xF5M>\x1A\x94\xD2\x97HXd8\x19\xAF\xA6\xB5\x04<\x94H|\x9A\xB0\xA7^M\xA4z\xBC\x0Fi7\xCDZG\xDB\xAC\xFC5_\xC7\xF5\xC2&2\x10`\x96\xE1\xE6\xE0\xA9{tB\x9Cd\xB8\x13\x0F\xED\xFE\x18\x98&\x1D1l\xD6S^\xFE\x96\x0C\xC9\x07m\xE0\x1A\xFE7\xBE\xFE\xECT\x92\xA6\xF7!\xF6p\xA5\x88\xA8\x82/45A\x88j&\xE6\xF7w\xCF5\xC8\xC5\xA5\xAEc8\xFBXPA\x97p\x1F.K\xE7\x93%<\xAD\x07\xD9<\xEA\xA3D\xD3\x91\xACg\xB
7\x0F\xF1#\xD6b\x7Fw\xB9=t\xC1?\xA7/Y\x81\x99\xB3\xD9\x1C\x9B\xFE\x92\xEFK\x16L\x0E6\xBF\xEF\xFD\xC8GgayE\xD5y\xD6v\xBC\xD6\x92&"f\x8E\x8ED`\xFC\xCE\x18\xB23,\x1C`\xEE\x9E>"t\x9EE&\x89\xB7c\x9E\xB0fL\xBF\xDBK\x80I\xAD\xAE\x13Q\xB1\x8Cl\x09\xADj\x0E\xA3\xA3\xB1\x89g\x08\xC6Iz\xA8\x96\xC3\xF0\xD5Mv\xEE\xF8\xEC\xCD\x81\xCA\x1F\x1E\xEF\x1C?\xEF "cG\x87\xE2\xD8\x0EV\xB6\xD4\x8F\xA5g@d\x1DuW\xD2n\xEE\xC8\x8B\xE6\xC3v\x04lG\x83-D\x0C\xBBP\x11\x02KH7L2od\x9B@\xB33\x1A\xD2v\xDD[\xC5\x85V\xAD\xEF\xDA\xAF\xD0_ph4I\xBD\x17\x9A\xFA\x8A\x9B\xAA\x1C\x1Fg\xF7a\x8F(w\x10O\xEB\xDA\xC6\x9A1\x0Du\xC3\xD2\xB2\xB4\x98\xA1\x83jN\xC5I\x06U\xD6H\x96\x86\xD4xGBR\xBCje\xDF\xAD\xC6\x0A\xDE(\xA2\x85\xA5\xD4\xF0n\xAAc\xF0\x13\x1A\xCE\xEA\x06|\xC9\xF0\xD4\xC3\xA0\xD8\xDA\xEF\x80\xE2\x8D\x12\xBE\x7F\xB7Da\x12#(l\xF7[a\xAFJ\x0Alhq\x9F\xEB\x9Ee4\xBF~\x0A\x02\x9F;+Mq\xEAL5Q\x9F\x8F\x1E\x141_sH3\xCF\xCC\xE0\x94\x94W\xC1\xBFn\xE4K\x8B\xCFJ\xE4J\x19\xCE\xFF\xA5\x12\xE3w\xD0\xF1\xACS_u\xD2\x05\xBE@\x0C\xF3\x10\xD5\xA8bk\
xD1M[R8\x8D\xA9\x8D\xB95Q\xE6M\x81*s\xC1N\x84\x89\xAE\x9A\xE2\xC0\xF7\x9B\xF41\xB9]s\xFE\x8C.\xD7\xED\x0E\x14\xB8\x8B\x93\xAC\x10b\xA2\xE9c3\xEFax;o9N\x80N\x12\x87\x8Bp\xC9b\xD7\x1F\xD0\x1CPwCB.\xD8\x01{\x1F\x82\x99 \x972\x13\xB8lVBZ\xD42+#\xE6\xD5(\xDBJ\xD4Vz^^\xE6\\x08\x7F\xE3\xE2R\xC8\xBB\xE1\xBA\xE1\xB6\xFC\xC2s\xDC<y\xFFU\x95\x85I|T\xA8\x07\x9D\xB7\xDB\xC1\x7F|/\x94D,\x8Da>-\x1ET\xF9\xB5\xA4\x13\xA6\xCF\xEF\xFE\x19{\xBF\x1D\xDF\x82N\x9D\xB9\x8A"\x1B\xD4\x00\xA48\xB1R:[J\x0B\x80"\x87\x84\x1B\x0B\xFE\x12\xEE9G\xCA\xE1\x8F~\xE2\xCF\x0FE!\xB0v^\xA7I\x17\xFE\xE1o\xF4%\x8A\xCB\xD5\xC5\xC1\x86\xAD\xD4\xA5_s;Ii\x87\xD71'\xD2\xAC\x9C~\x02\x8C\x16pP\xF7|\x0B\x8A!9\x14\xD7\xF5\xF5\x1D\xD6\x81O\xF6s\xDD\x13\x88\x95\xCF\xF4\xFB\xF2h\x8F/\x1DhT\xEB\x1DqX!x\xB1\xAF\xA2\xB2\x9E`C\xAF\xA0zJ\x07S\x19\xFC\xEE\x92\xF1\xF63\xB4\x84\x060\xDAN\xBC\xB1b\xBF\xBAJ@\x04\x11\x00wt\xA6!\x9D\xB5\x8BL\xE0\x9E5Z\xDE\xB4&\x07{w\xB3\xDD\xB5TI\x11\x82\x8D\xE4\xD7\x95Vw\x81\x11W6\x15r1\xC1<\xD1\xB9\xCDK\x81\x0Ct$
\x92X\xEDO\xA5\x9A&\xEF-\xB6\x0D\x1D\xE9F\xA8Kye\xE4\xED\x12E\x9A\x88\xC8\xD6L\xABq\x8B&\xA9\xE9\x8C\xDC\xADd\xB8n\xC5P\xBE\xB5\xF9\xD3x}\x09\x8F\xAC6\xEE\x1D\xEB\xCE/Q5A\xFA\xBF\x09y\xF3]mU\x05\xDD+\xAB\xB2N?k!i\x174\xACio\xAAi\xC0@\x10t\x1Aj\xA5\xD7\x17!>\xAC\xFB\xC5\xE20l\xD1L\xDB\xFDg\x1A\xDF\x9DN\xD9\x1A5\xA4\xE6\xF0(\x07\x89\x09\xD8\xDB\x96\x19>\xB2\x94\xB0\x08W\xA0\x84\x0Eq\xB4Q\x1Fw\xAB\x83\x12\xD0\xC6\xB2\x09v\xBE\xE1~'\xEE\xA6ZF\\xF7*_\xAC\xCD\xE7V\xE5K.1\xCB\xC6\xCCE\xF1\x99\x05P/\xAB\xFF\xC3\x03\x05\x1A)\x1FW\x178\xBC\xFD\xF3\x10\x9A\x80\xF9\xEC\xBEa\x94\x1E+\x85\xD5\xBB\xE79\xE7\x0A\xBB\xADQ\x9E)$\xA0\xBBV.\xFE\x96\xA3\xC7\xB6o<,\xF0\xB5:nT\xA8\xD7x\x13VB\x15\x93946\xBA\xCE\xB6\xC0\xC1/\xEBD3\x18\x1C\xB5\x1Al0w\xAAA\x81\x92\x96"\x86EW\xA7J\x17\xB1\x9EC\x8BWN\x88W\x8A\xFC\x9E\x19\xF7\xC9\x92\x1A\xCD\xBA\xD5\xC3\x8A\x95\xC0\x8D\xCFFv\xAD\xA8\xD5\x0E\xB2\x1D\x00=bJ\xCB\xE3\xD9\x9B<\x81\xBD\xCD\x89n\xA2\x90Oc\xF4n\xAA\xE6\xA4O\xB0\xDA\xCA>\x1A\xE02\x84\xA6\xFB\x7F\xF3)ZF8-^
k\xBE\x8B\xF6\xDA\x15\xE8\x0B\xED\xDF\xED+\xD7\x0F^\xD1\xDD\xC1B!\xB8\x86\xDF\xCE\xE8,\xFEy\x8B\xAF\xB0\xBFlW\xD7\x88*\xD8\x822\x86\xCC\xBEm)L6\xEC\xB0\x1E/e\xA2\xE8zvq\x1B\x16R\x1E\xA3\xCD}n\xFC\xC9 \xA4\xA7\xE2\xAE\xE0'\xF1X\x12\xC9\x80I\xD6qW\xDD\x9A1N/\xC3=\xED\xC3\xEE\x93-\xDF\xA6\xE7\x90-v]\x0E"N\x0FPdQ\x99h<?WD\x18\x89(\xB4\xCF$.8\xA6\xA3Z\x9FIo0\xB0\xB8\xEDu\xD2Z?\xB0\x16\x9B\xBA\xD8}\x85\xF08\x13\x10\x12W\x07\xF4\x9A$\\xB7H}\x93\x823\x1C\x80\xDCG\xF0=\xCA\xEA\x02u7\xC1\x175\x1B\xD5\x0D\xF6\x88z\x7F+;1\xA7\xF5_\xD8\xD0\xA2\x7F\xD8\xF8\xAD\x8A\xCB\xE1!\x08\x9D\x92\x11\x0D\xCB"g\x82\xF1W\xB2\xBF\x06\x90\x04\x93~\xFEQEKx\x04\xC7\xFC\xC1\xB6\xA2\xE5\x9E\x96g\xE5\xF8\x83\xC3\x96g\xCF\xB6y\x17!h1(\xFA\x94\xB6\xF4\xF3\xAB\x17\x1E\x18`\xB4\xDCmXA:w\x9B\x10e\xD8\xE0>z\x8D\xA3\x8B\xD6\xB2\xFBS5{\xF2\xD9\x1D\x10\xAAJ8\xE0S\xA2G|\xEA\x16_\xE3j\xA3\x85\x10\xA4\xA3"\xA6+\x90\xE7\xE4W\x9A\x85y\xF1\xD7g\x15*\xA5B\x82\xBD\x91\xB7\xDE\xA8*9\xBDu\xC7m\x9F\xAFE\xEA\x82\xC2\x84n\xCE\xAC[q\xE8\xF
8\xB2\xF9\x1F\xBDf\x1D\xC8j\xF0:aK62-\x09\xDC\x9C\xEE\x83\xDBYl\x94\xAF\xD3\x9A\xBB2\xE4A-\xFAx\x81=\xB0*\xA1\x0C\xCD(+\x98\xB6\xC9\x1E\x04\x9D"\xDB\xC9+\xF5\x16?\x80\xCC\x06eo \x02\x1D*\xA5BA\xA8W\x9E\xF7:R2ZW\xB1\xBD\xCB\x91\xF65m\xF6\x8A\x12\x1A\x9F=\xE2\x87\xBB\x90,}\xC6\xD8\x9C\\xCB\xF9ut=\xA7\xC2\xA7w\xA5\x00P\xB6\xFDA\x87\xF6\x9DY\x0Fm\xD7\x94/\x0D\xDD\xC4\xF5>h~4\x12L\x82\xA0\x9C\xC2\xFBx\x80.\x1D \x90\xE58\x19G\x83\xC8\x03vt\x97\x12@AR\xFE\xF0\x06\x0Ba\xD7bA\xDEE\x1AQ\x8E\x07\x95\x0E\x8F!\xF0\xC6\x96\xB4Q\x9F\xCF\x97\xEF\xF7`\xF0\xD4e\xCA\xD8R\x19 \xA7\xA0\xD5\xD5\x19\xE4\x0A\xA7\xF8\xD7\x14\xE8l\x08\x8D@\xC5\xE4\x00\xC7\xDF]\xD6\xE2p\x0D\x0CF\xC3g\xB3\xD7b\x11\x85\x05J\x19\xC3\x07?a1\xB3y\xFB\xA0N\x82\x05v\xD2`V\xEBf\xF6C^"\x13\xE3\x98hb\x949\xE2\xA7d\xECP\xA9\x8A\xAC\x09\x0E~\x08\xEBKBk\x17\x848\xFE\x80Rs\xF4\xF1j\x11\xB9P}i\x01\x88\x872G\xA5\x01\xC2\x91\xFA\x8D\xBBC\xFCA\x1E\xB00\x96/m\x98\xD7\xB1jU\xCE\x17\xBF\x9D%\x88\x8E\xFA\x17\x8Bm \x94@/\xFF\xB2\x86\xBF\xBF\x0B\x08
\xF9\xF6\x92\x08\xABF@%\xE17\x0C\xCA\x82\xAF\xFE"\xB7\xF5R\xDAI\x9D\x9D\xD7\x98\x7F\x0D\xE9d4\x08\xF7^\xD9q\xCD\xBB\x05\xB4O\x80\x1B\x84\x88\x93\xCA\x99\xDD\xF6M\xE6J\x9B\xCC\H$g\x82\xBA\x07U1\xA5\xDC\xC5t\x7F\xB9.C\x9E\xEEu\x98\xEF\xB3zB\x85\xE9\xE0\xB8\xE3v\x9B\xBF\xFB\xEE\xC86\x1E\xF8"<\x1B\xD2\xA1;8QHt\x12\xDA\x1C`[Z\xC5\x04\x91\xD1\x15\xED)\xC2\xE9\xE6\xB2U\x94\x9B>u\x80\xBF\xAB\x85R\x9EoF\xA5\xBC\xB9t\xC2hB1,WH\x16\xEE\x82\xA3\xBB\x90\xFA\xF6}Y\x8Bf4\xDBk\x89\x9CU\x8F"\xEDR\xFEf\xF1\x81\x9C2\x98\x7F\x123\xC1\xA5\xFB8p\xB9\x07\xA4\xCC\x1D\xAC\xA1\xDD\xA6\x9D\xDB\xA4\xE9\x88\x1D\xAFWe\xCB\x0A|\x9C\xCB!\x81\xA1`\x00\xE4\xA0\x1EE\x83\xE3\xE7m@\x87b\x8E\xA7\xD4\x93\x8D\xC6F\xBB\x8BA>\x9D}\x83"j\x13V\x92'\xA5{Q\xA0q\xFAKs\xD0n\x09IU\x0C\xDAM+f\xCE\xFE\x91\x04\xA0\xF4\x8B\xD9;X\xE8\xC9\xF39\xA8\xBB\xF5\xE4\x81^\xC1h(\xA9\\x08\xB9Y\x9F3\xB6\xDC\x07\x0Eh)\x03gy\xFAn>\xCF`(N\xA1O\xFD\x80\xD3\xB8\xD1\xC9\xEBVN\x972$ 5JAi\xB8\xC9K \xB4\x0Fz\x1EB*:\x19'N\xB0\x88y\x1C\xC3\xD0\xC0Vk\xB3J\xAC
\x9A\xD0\xE3h\xA6v\xE3\x01\xE9\xAF\x07\x1C\xF0\xBA\xB5\x11\xC3\xF2y\x13$ ,\xE3\x1D6*\xE7\x99|5\x09\xE1b\xE8s\xA2\x18ms~\xE5\xEC\xEF\xF4\x95\x7F\xCE8\xD5\x0AuO\x1Ai\x03\xD7\x1B.\xE1\x9Al.\x9C:\xEDX4h\x0BR\x14\xF8\xBF\x11\xDF-hd_\xFB"]\x9EU\x14\xF1-@\x97\x8F\xCB\x7F\xA2'\xAE\x04\xBB,Wo\xE8\xAC\xEDn\x89\xE1\xC5i\xDC2\xF8\xAFV\x7F\xDD\x88\x1Dpc\x15\xE6\x94\xF1\x17j\x18\xFC\x80R\xAA\xEC\xAC\x8Cp\x9D7 \xCF}\xCD\xB1e~3\x8B\xCD#\xABx\x03~\xF3\xA9\x10\xE2d\x04\x967py\xF3\xE6[\xDD\x96\xD4@\xE3\xE0i\x10G2\x0C]\x1F\x01\x86\xBE\x14Q\xAA\x1E\xAC!\xF0\x1EZ\\x83|B\xC2\x9FU-\xDB\xD9\xC1[\xB1\xED\xC8\xA0!r\x90\x1F\x87\xC1e\x1F:\xE5\x98\xE3\xB3\xB3I\xCAs\xF8sm\x01R\xFA\xC4\xFD\x059\xFE\xA0\xF4\xC8<B\xB1 _\xE2\xB2\xD4R<\x83\x88\xB7\xAA\x86\xD8\x86U$S\xA4)\xA0\xA9N\xFCF\xBA\x9Cfi\x8AD\xB9\x80\x82k\xD1\xDBP\xE1\xA7\xAD\xEE\xB7\xE1\x09c1l\x88\x1F_4\x0C\xCB)\xA1\x9D\x8Dd\x85\xBF\xAB\xF0\xF9\xCAN\xC3\x0B\xE3A \xEA\xDD\xADF\xDE\x9D\xE9\xA9\x92\x16\x92x\x84\x7Ff\xE6I\xA0\x1D\x81pj|/2.\x15\xFB\xAC\x12\xD2TOc\x
E0\xEA!\xEB\xA9A\xBF\xBA\x91\xBD[\xC4Lk\xCB\xF95\xBEMd\x99\xE0\x8B~\xFE\xB9\xCF\xD8\x1ED\xC8\xE3~H\x8A^\x03\xC4\xDD\xCE}\x01T\x0D\xD0\x88\xE8\xE5\x91\xC3Y\xA9\x997\xBA@?\xB8mF\xF9hr\xD5V\xF6\x10\xA4k\x07\xF7\xA3H\x81\xD9l\x0C\xFA\xE5\xB5\xBA\xC3\xE9\xBB\xCA\x17\xEB0Z)U\x936\xF1\xDB\x04Q\xAC\x8A0s\xF8(\xFFa\x9F\xD3[\xC0\x16eP6\xB8\xE7\x1E\xFDz\x9C\xBC2<\xE7\x98\x90\xE4>\x10\x08\x84\xBA\xBB\xD9P\x9E\x9AqB\x805\xB5\xB7\xD4\xB0\x05\x85o\xDD`5\x06\x86\xEF\x9A^U\x16\x179\xEC\xAEE+\xBF\xB2l\xA9\xA0\xE1`\x05\xBA\xB7\x0Fv`s\x9D\xE4\x04r\xA8\xC0\xE6\xAC\x14\x90j,\xC9\xD8`G\xFB\xC5\xFB\x99\xEFa29\x83c\x8F {)\x052\x04q#\xF56\x11f\xFF\xDAr\x9E\x1D\x170\x87(D\x0A\xF2\xF7+\xA9\xE5FGC\xBC\xEC\x87\xF5\xD2\xCA\xC0_\xD2\x03\x8C\xCDDF\xFD@\;\x9A\x03R\xD5""kV~EA\x1B.\x8F,f\xB8\x80\x12\$.')\x07\x17\xF3\xCD\x0B\xAC1\x8E\xDD\x92\x04#?\xC8\xF2\xC4\x0E\xD9C\xD3>\xEA0\\xC2\x84\xA2\xA7w\x80A\x8E\xDFG\x9A\xC2\x0CuP\x16\x88\xEB\xB5\xAF\xD6\x1CVM\xAA\xD4\xDB2H\xDB\xFC\xEBL\x98sq\xEC_\xC16\xD0K\x93\xF2,\xB2\x0E\x9
C^\xB2\xCF\xA8\xD5\x12)x\xA6+o\x0F\xCB\xF3)\xD9n\x8A\x0D.\xECHLM_F\xEE`\xD6/e\x10\x7F\xCF\x94\xE1\xD1\x13C\xE1\x00zV\x8A\xD7P2vo\xF1\x07\x01\x1E\x9A\x0A\x80\x12\xD9\xEB\xA7\x16\xA6u}w\xF5\x959\xE2\xD6@\xDC,\x82^\x1Bg\x93\xF7\x1C\x8C|\x90\x0B\xBE\xE5*m"\x94\x84+\x7F\xEE\xFF\x86\x1A\x8D<S\x8C+\xFD(AdGk\xFD\xD4<7\xC4\x0E\xE9\x0A \x80\\x1F\x7F\x1C\xE1\x1B\xE9\x01\xE9z3*D\x9AB\xA7\x1F\xB8w\xB4F\xFE\x99,+\xD7\xED\xA6\xE9FwgU\x9A7Z\xE1G\xC7H\xBC\xF1{O;\x9A,\xB9\x8C\x7F1\xA3\x83\xAED\x0F\xCE\xDF\x94\x9Fx\xD0=JOw\x96\x92\x98\xBF\xEF\xB3)\xFFt\xF2\x07\xA53\xA5\xEF^\xF0\xBC\x9B\xF6\xBE6\x96\xBDf\xDF\x14\x10\xD5\x02\x86\x91\x85\xA9\xCBF\x07'\x80\xF2\x05\xE2^\xC9\x8DL8\x14\xF9d\x1E\x0C$\xC0\xD7\xAE,)O\xA49X\xBC\xC2\x0A\xE6v\x97\xF8\xF87\xA7\x02\xEDL\x19Aw\x1A@\x8F\x11\x12\xCA\x81\xE3I\x9C\x96\xD8N\x90\xE44\xB5\xAF\xEF}Ft!\xB2ie\xFBg\xB1@Y\x87\x8Cz\xFD|nk\xB4\xFC\x89\x81X\xFFK\xBA3\xCE\xBC\xA4\xE6\xDC \x8B\x1F\xBBTR\xF8{\xC0k\xC9\x1BIw\xDBr\xA0\x15\x0F\x9D\x95e\\x04\x15\xBA\xF3g\x1D\x82\xBE\x13\x
98'gb\xDAp\x02z\x82}4.O:D\x9D\x083\x9B\x04T\xE3\x0A\x9D\x8B\xF7\xE4x\xA9\xB8w\x1C{\xB5\x0Dz\x012\x9CIQ\x96\x15\xA4d\\xEE\xB9z\x9D\xF8\xB0\x0CB\xEC6%\x19V\xF65VL\xA8E\x13r\xD2s\x95\x8B\xEA]\xEA\x02!\x8C_X\xEB\xC0\x98(\xA7\xEA\x1D$\xAEk\xC2-v\x1A"\xFD\xFC\xAB\xB4\xAC\x01\xFF\xD9\x97\xEC*\xE8\x04%\x94\x13{n\x16\x9D\xCD\xBC~\xA0\x89{{\xA2\xFD\xA7\xB7\xC9\xF5\xFE\x13\x95\xF1:\x16\xBE\x1C\xB2\x10\xAF;'\x8B\x0A\xB4\xD9\xAC\xF1\xA9l<I^[\x87\xD2\xF8^\xEA\x1C\x1A\xE6\x17\xFAMtNP\x95L\xBA\xD5LjC\x1D\xDA-(\x17g\xB1\xFCE\xE5\xCBU\x84\xE7\x07\x9BZNC\x06\x06l\xBC\xE3O\x80\xC4\xA8\x80S\xBD\xD4\xFF\xADn!\x02\xF0\xF8z[]\xA9\xBD\x00r\xEF2\x8D\x0C\xEDc\x1AFm1\x1D@\xA8\xAD\xC8L9\x1A\xE6\x18,\xA5%\xE8\x81\xD6]\x1F\x9B/\xF6\.\x1C\xF7\x98!d\x0A\xF4\xDB;:\xD8\xCFqq:P28\xEBa\xB4M\x09\\x169\xE3\xFB*\x17\x91jI\x8F\xA7\xADt\x1D\xFC@'[\xAA]\xF0\xEFu0\x01W\x0DX\xC2Ir\xD3\xA0\x89me\xC6\x08\x08\xDD\xB1\x8D\x84.\xB1\xE1\xD1\xD7\xC0\x06\x95&\xF9\xC0\xED\x93\xADpT\xB99g0F\xC0\xC4'\x0E+\xA1\xF85\xCA\x07\x12\x97n\xCEl\x
9F\xF3\xC6\x15\xD6\xAF\x0FG\x90"em\xEB\xF5\xBB\x8DB(\xF1d\xB5\x18v\xDC\x09\x14\x9D,\xF7\xB3\xB6 \xB9R\xE9PU\x0C\xAC\x1D\xE2\x8E\xBA\x8F\xCE\xC6@\xEESj\xEFT\xF1]\xBB(k\x8D3I%\xB8\x0B"\x1E\x94\xCF 6YQ\xDCz\xE5\xA1\xEAo\xF1\xBBE\x13\x80R\xE8Oaq\xD45=\xC4]Rn\xEF_G\x13\xD0\x8E%\xB8<\xE7\xCD]\x0B\xFE\xE5\xB0{p\xE7E\xC8&\xE2\x9C\xE3\x05[\xAC7g\xC0\xF6\x16\x9EnL\xA6W\xB3\x9A\x13\x13w\xF3\xCAQ\x82\xAD\xE0\xF8`\xFA\x11\xBAw\x1Ew\x81,gR\x1E\xCC'}\xF8Bcs]\x0EI\x1A\xA2Y\xE5\xCC<\xC0\x8B\xB8\x13D\xC2\x8C\xAF\x06\x07\xC9\xB3@\x7F\xB1\xAD:\x0C\xE5:F\x9CN)\xB0_\xBAAD6Z\xAE\xDC]h\xDD"%J\xB2y\x0D\x1Dq\x09\x09\xD7zDd,\xB6\x8B\xE3e\xDESMY\xC2{.m+\xCF-\xE7!\xE1\x91\xB8?e\xF9\x1B\xC4\xA0\x9FFT&!n}<E\xAE@\xFF\xD6~\x83K\xD7\x15rpw\x889\x9A\xD4\xDB\x98\xF2*\xE2e\xA0\xDD\xB1\x96\x1A\xE9\xF44\x18u$\xB1V\x91K\xCE\xDF\x81\xEEt\xF7\xAB\x96\xC4(\xA4\x15r\xA5\x10^\xB9\x11\xE54\x1B|\x98\xFD\xDC$H\x9DA\xEFj?\x98;>\xA0\xA0\x99\xAE\xBDO\xFF\x17~\xA3\x7F\xB2\xFEc\xA5\xDC\xA2\x00}\xBCIB\xE39&B\xD4!Ph\x10gw\x9D\x07YE+\x18
+\xCC[l)\xB2\xDDjj\xE2\xA1K\x18\x82\xFE\xA8\xE6\xF5\xE9\xFF\xC9OX\x9C\x15\x08\xFF\xB3\xF9\x1B\xFF\x81?\xCE\x97\xCA"4\x921>ZK\xB8!i24\x90\xEC\xFD\x18K\xA3lw\xA9\xE2\x93M\xCD\xB0\xE5u\x98r\xE0d`\xACm\x96\x88m\x81n\x84\xDDc(\x9F\xA4\x96a\xE1H\xBCfDXG\x9B\x8C\x9E\x17\xF3\xDE_\x91"8\x9D\xEE\x81}\x08\xEA\x9C\x9E\xAC)\xE7\x1C\x13he\x9B.#\xE1OO\xC7\xF3\xDC%r\x93\x9C\x1D*E\x14"\x17\x84\xD7<+w\x025\x8F"6\xB8,\xA0\xDD\xD8x\xEE\xF5'`\xDF\xDCwb\xAB\xC3\xCCk\x12\xF1}\xD7\x0CR\xEEPU\xF7\xD8Y\x9A\xBE_\x16\x07\xDFX\x85o\xA4\xB0\xC0\x0E\x9B\xD0\xFF\xCE'pEb\xE0\xF9\xCB\xF80\x8F\x8BT\xF8@\x9A\xB0\xAAqf\x1A,\x93.\xAA\x90*\x98\xB5\x9Ea^\x85\x00\xC5\xD0\xA5\x88\x13hK\xB6\xECYE2n3\xE5\x95\xD2\x8F\xE6#\xD6\x97*0^@\x84^\x0CeJk\xAB\xE1\xA2\xA2\xFB\xE88\xAD\x0Bpc\x09\x9DA\xD3=T\x84&y\xC0-\xA7\xBF\xBF\xE5\xEF\xC8\xEC\xAF\xFE\xE7\x1E\xEAM|d\xA9\xD3\xE2\x14\xED\xCB\x93\xF7\x8A\xFE\x05\xB6\x0F\x8B\xB5w\xD8\x9A\xBC\xA3\xDF4\xBB\xB3I\x8C\xD9\xBE4\xE2w\xA0\xB0\xC9\xA6Q\xB2\xE6\x1B\x18d\x9Aq5b\x05\xB3.\xCE\xB5j\x83*\x
83\x8D\xD1E\xD6\x08\xE7\xB7\x96\x89\x10\x97\xA0N8\xED\xFF\x9F[\xB2v\xAC\xAE,\xD5"\xFC\xBF\x1AQQ6\x11\x87p\x85\x00!X\x03\x14\xBC\xE2\x19\xC7\xB4\x88&\x1FS\x04\xF6j\x05M\xF0\x04\x06\x01[\x9B\xBB\xDDzed?1\xBC4p\x0A\x1E\xAD \xF8\xA4\xB4xh\xC1\x1C,\x86\xEAm\xCBc\xF16\xDFW\xCC8K\x84\x03\xD6\x0B\xB2\x8C5\xF3\xCC\xBA\x0E\x0E\xF3\x8A\xB0\xF0M|\x16\x87h\xE8\xFBD\x10s\xDF\x7F\x92\xD6\x1A\x93i\xCD\x01-2\xE8|c }$\xD7\x0B\x06\x0D\xD8\x0C\xB8\x0F\xF6n\xD4h\x95\x87C\xFF)\xE4_\xFA\x19\xDF\xD6\xFE\xBD\xF1\xADE\xEA\x1B\x81I\x8FX\xA2\xB4q\xCD\x9E\x09\xE9\x84F\x94I\xD6A\xEB\xE8\xB9\xC31TW\x1BZ\xA8\xF7\xAB1\xD0\xE8\x15k\x7F\x0A\x9B\x92\xC8\xE0\x17\xBC\x91V\xE0\xFF\x1F\x13L\xC9B\xA1\x18`z\xE3\x84\x94~\xE9\xD4\xC3\x85\xF9\x06\x8C\xC4\x7FN\x18,\xA8%\x11\xE6\xBB\x08P\x19.~\x17?E-\xE9M\xC6\xF5\xA2&\x14V\xE4;\xE9\xF4@>z\x8E\xBB\xBC?E\xA8r\x91g\xD2\x18a\xC8"\xBD\x95\xBCs\xED\xEA\x92lT\xEF\xF4`\x0C\x8642\xCD\xD5\x0Du\x08>\x7FS\xFD\xE8s\xF2\x14\xE4\xF9\x90\x1A\x13\x96i\xB9\xB1aP\xD4\xA5\xFAg\xB2\xC78\x9F\xE1W>B\x
B7\x1A\xB1\x99\xF7\xBB\xF9\xC2\xD7\xC6\x8E\xD7)x\|H\xC5g\x9D\xA5l{\x0D\xD3\xB4\xA2\xD9Yr\xEA\xA4G\x0E74\xC1\x897yO\x12\xCA\xC3x<\xCA\x1Dl\xE9\xDF\x18\xE5\xB2\xF6\xCA\xD5x\xB0\xC5\xB0\x13\xB8 \xDB|\x86\x93\xBA\x8B\x17\xF1n\x93\xE7\xF4\xE6\xA8 \x04E\xF6\xC2z\x87t$\xFAL,\xEB]|\xDE\x8A\xBF\x12>yY'i\x13(\x9E\xF1z]\x8Cx\x83\xACgq\xD8\xEF\xE39C]\xEE\x11\xA10<bm\x8E\xFBS\x02\xDC\xE0\x94\x84\xBF\xBA\xF2c\xC6\xEF?\x88\xD4\x10w\xBE\x98\x0E\x8Ea\x99\x1BJ\xF0\xDDw/B\x1E\x0B\xE3\xC4\xA7\xD8\xBE~~\xA8\x0E\x1AK\x8D\x14.\xFFi\xE5\xDC\x0A\x03\xF6\xEC\xB3\xFA>\xE4\x9B-\xE1\xFE\x04,}\x8E\xF8j\x03\xD5\xCEq\xB4s\xDC\xAC.\xC7\xF5\xDD\xFB\x03\xCD\x9E\xB2G\x9A\x1EKn\xB4\x90u\x9CZ\xDA\x11\x06B\x81\xC2\x95=\xF8\xE2w\xBB\x88\x7F\x1B@}\xE0\x8A\xDFk\xD7E\x9C\xBF\\xD0\x06\x18\x909<P\x15\xF9\xAD\x84H\x1C\xFE4\xF7:\x01\xC4u0\xB0r\xBD\x1E.;\x809\xEF\x83\x97f\x81\xDC~$L\xC2\xB0\x90\x91 \xB3\xD2f\x09\xD2A\x181j\xAF\xD8\x87h\x17!\x8F/\xA5*\xD86\x9B\xABG\x1D\xA4/\xAC\x0Da\xFF\xD4\xBB\xE1p\x9F\xB7hB\xE8n\xD9\x8E\xB9\xC6\
x12\x10\xD6\x9F\xC2\xC9\xD726\x1F\x12\xEE_\x0E-\x0E\xB58YRW:Z\x8A\xD5\x11\xAEzTn\xDC]\xCC\xC7t\x91\xCB\xAC\xC5i(\x09\x92\x069\x16t\x04\xB13\xCE0\xC0\x86VC\x0F\xED\xA9*\xE3XQ\xC6\xA2\x08\xA1_Y\x98\x14Ph\xACY\xA5>\xF8<;\xE7\x1B\xC1\xD3\xCB:{\xCA\xEE\x92l\x94E\xEE\xC9\xEE\x00\x05\x0Cf?\xB4\xFFht\xDC%\x8E*j\a\xC1\xCF\xFD\x7FU\xE0\x19\x90\xBF\x8EF\xE2\x16\x17\x1Dd\xED\xEA\\x81\xFB\x0E\xB3\x09\x96\x84\x1C\x85z\xA3\*\xB2a1@\x18'\x1F\x8A\x90\xC7\xEE\xB3x\x7F+\x0FVQ{47\x14\x04)\x0D"[\x83\xCD\xE3)\xB6z\x98\\x04g`\xD3vh\xF4x}\xC2\x14\xD4\x1AK\xCB\xA9<\xDB\x1B\xBA=\xE4F\xD5~\xA4\xE9\xCE\x91\xEFn\x16M}\x02+\x87\xCA\x1F\x9C^8\x8A\xF8\xF3?^p\x9E\xE6\xDCfmI\x17\x83\xDF\x07{Ac\x91\x91\x9E\x0C`&\xA7\xD2\xBA)I,\x02\xD8\x91\x95\x95\x02Wu\x91\xF2\xCE\x8D\xF0\xCD\xDF\xFB\xDB\xE2>\x92\xFD\xE6\xBF\xA7\xD1\x0B\xC66<F\x91\xA68\xB9\xD4\x92%#p\x10C]\xE2\xBA/8\xB86\xE0k8\x80\xB3\x02\x83@\x8F\x93\x03\x84\x13}\xDC\x14\xE4-f\x91A"\xB3%\x8A\x01E\x83>8I\x98m\xE7O!\xEB\xAF\xB0\xB4\xE8\xC4i\x1Av)p\xC24\xF5\x9F%\x9F\xD
4&&\x95(g\xED=B\xAE\xFC\x0A+d\x08Bn$\xFA\xC7I\xB1W,{E\x15\x13\x1A\xE5B\xF4\x01\xDC\xFC\x9F\xFC-\xD7u\xE2\x9F1\xD6\x91-8\xD4\xEF\xEE\xC2g\xEF\xF1x\x9A\xF5/9bf\x06\xB2\x9C\xE6\xB7\xC2k\x0E,r\x823\x82\x9D\x92vjK\xEE\x9AH\x93L(\xE3\xD1\x14\xF0S\x99\xF5\xE9\xB6?}\xB4\x0E1\x1C\xD2i\xDE\x02h\xC3I\xE7\xBA\x1B\xC6\xE6\x09T\xB3\x8E\x8B\xAC{\xDE\x17\xED\xA1Fv\x10\x95}\x0E\x0E\xDD\xC5 \xBE%\xB8G\x8Aj\xF7C\xD1\xA1\xD4\x8B\xB0\xA6&RF\xDD3\xEAd\x84m#(\xEC]\x8F\xF7\x85\x8F\xB5\xB6\x19\xEA\xA9\xB4\xBD\x09z\x06g\xC9\xA3\xDC\x13\xB9\xDAi\xAF\xCC<\xB9\xF7\xBC\xCD\xB2\x12bB+\x00\x88\xB0\x10\xD6U\xFC%[d\xCC\x8Ee\xED\x88hER'\x12^S\xB62/\xA6\x91\xB9lG(\x02\x879\x15\xC1\xE9\xDF\xC1e\xD0\xDB*\x12\xDE\\xA3\xF8\xAE\x9C]\x8C\xE9iRk\x817T^\xE9?\xB6x\x7F\x92M\x8Dh\xC2\x0E\x8E\xA1\xD5\x89\x99.\xED\x13\x18 aH\x18\xA9\xF6\xE0\xA4y\x9D=\xB0\x1B8T\xA9\xF5\xF2\xA43g\xF5\x0C\x8A\x89\xA7c+T\x11,\xDB\x8E\x8A\xD5\x18\x91\x9D\x84oM\x8B5\x90\xE9\x82\xDF\x16_\x0D\xCA#\xA4\x1Ez\xA4\x13N5xS\xED\xF3Q\xA4z\x97\xD4\xDE|6\xA0\xF0Tn
\xA5\x82!'\xC7k\xD0S\xC7^k(fhn\xFF\x96ZH\xA4\xCAt7C\xC6K~@\x97A\xD2\xD9a\x91Br-\xF0\xBE7Z\x9FU}\xC4gx\xC7\xD1\xADu-\xE4\x0D\xDE4}*\xD0\xC4\xD6\x07\xD8\xAE\x16\xD8?\xF7\xE41\xF2RSq|\xF2\x9Fj\x87\x16\x82\xB1|\xB0p\xC4\xD0\xE7\x0D\x0Fd4\xE9\xBE0\xE8\x18\xACUd;!\x82\xE5\\xD2d\x95\x92EQy\xFD\xA2\xC0\x05(lK\x80\x12\xE5\xA2\xD8\x97g\x84\xC6\xF5\x00\x8CY\x8BI6\x13\xDDW/\x87\x0Eg\x0C\x01\x92\xD2(5*\xE3bD;\x95\xE0Kq\xD9\x8BL\xB7\xF5\xCC\xEBf\x821e\x0E\xCD\x0C~\xCC*\xB7\xC3\x92pR\x00R\x9B\xF8\x1Do\x9C{\xFE\x9FNA\xC08\x16\xF1"\x92\xAB\xC1G\xA8K\x82D\x8A\xBC\x93Yt?\xA5XW\x0B\xD6\x19|6\xFF\xB4\x10$}-\x8B\xC2\x9D{\xCD\xFD\x9B\x89\x17\x91[6\x19$\xFD`\xF9\xBC<\xE4\xC11\xF7\xEBW\xFE\x10\x04\xE9\x07\x93\xE3in.fp\xBA\x95\xDEF\x92\x80\xC5\xBCT\xF3\xF8n\x94w~\xAB\x93\x06i\xF0%m\xDBM\x10\x11\xF5\xBE\xAA\x85\xC9\xD3CX`\xF7\x8F\xC1\xF8\xD8Y\xDDG\xF9k\xA7\xB5Ra0\x0C\x8C\xA7\xB6\xF7yt\xB7\xB97+\x8F{\x8C\xE8\xA6\x0A\xA9-\x13\xE9\xF0\xFA\x09\xF6\xEF^\xCBL}\xAA\xBDs\xBA\x12km$\xE2\x1CL\xE99p\xCAu\xEB\x8B=\x13M\x
A2D\xF5ph\xB2J\xB7w\xB9\xAA\xCC[\xE0\xD4\xD6\xDB#7\xA98\x0C\x05K\xE0\xF6\x8B\x92)<-\xB0\x9D\xCB\xE2\x06\xCC\xAC\x1D\xB8\x0D\xA7\x16\x85\x8F\xB9\x0E-\xA9W\xD579\x1C\x9D\xA2\xE7\xB3\x96F\xFB\xAF\xC6\xA3\x91\xD4\xB0\xBF\xCE\x1A&\x8B.t\x8BP\x96\x08\x16\x04\xDD\x16\xB2\xA0\xA2\xE3x>N\xAB\xA3\x0D\x0A$\xB04\xB8\xC9\x8F[\x10\xB4f\xA3\xE4\xB4\x04\x03\xE4\xFCC\x8Fn\xC6\xB6%\xAE\x8D\xF9uY\xFAP\xEA\xF6\x89s\xB1\xAE\xA7wV 7\xD4&W\xAC!\xC1}f][\xA9\x9C\xF9\x00I\x88\x0E\x02\x13\xB0\xE7\xE9\xC4\xB4\x9Cz\x97\x80\x06\xBA\xDA\xE6\x111\xFB\xD7g\xC5\xAE=\xE3\x0A\xAD'\x04\xE6\xDD\x09:_8SF\xEB\xFB\xDB\x91c\xCD\xF9\xAE\xDC`\x04>\x01\x1FeU\x9B\xF6\x1A\xF3\x1B\xFD\xC3i\xF5I\x85\xA5\xBB\x1DZ\x9Bh\x14\x1C\xFC%\xF4\xCC6h [\xCEd[\xB7C\xBA\x9B\xBD\xB7Y\xE2'\xAE\x1C\xFB8F\xE3C\xB3\xEF\xCD\x1B\xF6\xBF\xA9_G\x85A\x8C\xB8|\xF1\xEF\xE5\x19\xAA\x00\x86i\xDB1\xD8{\x1C;\xA1\x11\xDC\x1C=\x81&(\xCE\x85#\x82\xCC\xC1+DlC\xE0N8\xCF\x1B\xB5\xFC\xDE\xB2\xE7\xB3*\xAF\x17\x04B$\x7FO\xED*\x81l\xD2KOO\xC2@i\xCDi\x9B%\xB0\xBD\x9F?J\x
AD\xAC\xE5g\x96\x89-\xE7\xAA\xC4\x93/+\xFA[{\xAF0I\xDB\xC9\x98\xC6\xB3\xA0Q\xFF\x7F\xC4\xE3\xC7\xBD\xD8\xC3\x10\xF8D0\xD9\x95\x1D\xA9\x9F\x9CV\x07\xA6\x0C\xE4c9\x0D\x97\xBBF\xFB. 7\x8B:\xA3\x1CG\x15+\xFA>\xDD\xDA\xF5\x9E\xDD\xF0\xE4EJ\x06\xA3\x13\xBD\x87o\x97\xC2\xB4\xE7\x07.\xA6O\xF0\xB0\xB2\x0B\xFF\x96\x0D;d\xED\xDF_U%)\xFDP\x0A\xE1\x10\xE9e|\xBC\xBFm\xE5\x00<\xD6\x12\xEE|\x85\xD1\xA5\x9Fri\xACi\x01c\xE5Q\xBA\xA3y\xD4\xFE\xFF\x1B\xC8\xD0\xAFG\xDD@\xED6\x84\xDB\x00\xA72E+\xD3\x05\x8E\xA5\xAE\xA2\x09?-\xEC\x91[K_\xFEn\xADR/R\xCA\xE0\xB9D\xEB\xCA\xD9\xB0\x83\xC7\xDE\xB2\x90\xB2 \xC5\xC66\x11ga\xC7\x85\x19pp(\xA9\x83\x0B\xFBp\xB2\xCA\xCAbN\x80\xD1t\xA6(\xA5\xC1IN\xC4Z\xF1\xA1D7\x17D\x18\xCB\xA6\xFE\x8F\x15\x16\xDA\x8D:./\xC8\x87|\x1B\xB4^\xBD\xF2!\xA4\x04"\x15\x9D-1Hp\x05\x0AI=y\xEF\xB5~z\x8CJ\xEC%\xBB\xA7\x876\x96\x00\x84\x02\xF9\xB7\xFE\x83\x9C\xE2>\xED\x8C\xDE\xD6\x94\x85y\xAD\xBD\xA2\xD3\x0EcQU-\xB9\xA9\x8E-\xD9\xEBoha\xC2\x154\x12\xCF\xEF\xECG(\xDF\xB1=da\xE2\xCEf\xF5\x9A\xD2=\xD
C\xD7\xEAL\xB0\x12/\x9D\xB1a\xF9\x08x7g\xF0C\x93\x0B\xD2|\xA0\xF1\x19\x13\xBF\x1D\x00\xE5FT\xC8\xEFr\xC4\x8A|\x02\x9A28\x06\x9D:\xBCt\xFC\xDA\x0A\xCC\x04(\xDC\x82\x86\\x10\x82k)\xE4\xCBZO\xEB\xDE`[\x1EF\xDA\xB1\x15\xF9\x1E\xFBB\xB8-\x8B\x05\x1A%\x1F\xFAn4\xA9n"\xF0\x0E7\xEC\xEA\x91m\xF6\xFBOJqk(\xE8\xA44\xA2\xA2@\xB9\xA5\x8A\xD0~}\xE2\xEB\xBD\xBD\x0B}\xEC\xCE~\xDD\xDB2\xE1%[\x05]\x8A\x07(\xD6\xB7w\xB8)d\x8C\xEE\x16\xCBh_6\xED\xC6\x1D\xD2C\xAC\x00\xE4\xB3\xC7;\x95;\xA1\xB5\xC9\xA7bG\x9Br\x97|1tM\xD0mV~\xB5\x02I4\x8F\x0D\xE1\xC7*\x11\x18=\x98e\xF2\xE5\xA7\xD8\\x9A\x9D\xA4\x1C\x1Bb@\xC8\xBE\xD9I\xED\x99\xC1\xE6\xA1*\x9E\xCF\xC0\x8AK\xFD\xD1\xE2\x99\xF3\xF8\xB7\xBA\xB5\x8F%G=//\x8F\xFC\x1A\xEA<\x9D\xAA[\x9C\x8A\xFFh\x1Ad\xB2\xBER\x86\xF2\x04qnw\x06\xDE\xCE\x92J\xE8\xCA\x09\xD7_\xEE\xA2\xFEOb\x93F\xB1e\x8FH\x96\xD5S\xC1\xA4\xDB\xA5r\xBA\xEBU\xCD\x9A\x82\x9BB#\x1AA\x85\xE0\x12\x91\xEF\x0B\xFC\x19\x02@\xD5G\xA0\xE9\x81\xB9\xB4\x9A\xB8\xF8\xAA\xF0O\xC5E\x87\x1E\xF0:"\xBB\xDB\x8CT>\xDF+\x8F&
}\xD7\xDC\x82g(\xAD\xBE\xC3\xD0\x00o\x1A\x16\xF1\xC6\xCCJ\xD2-gU\x12n\x0D\x16\xB3N\x98;\x9Ew\xE4\x14\xEE\x1A\x0B\xBF\xC3\x8D\xD4V\x98\xC1\xE5o\x83\xCC\xDD\x17\xBE\x17\x12\xA3\xA4\x9FCA&.\x04\xAA\x1Co\\x9A0SS\x1C\xCDa\xD7m\x95\x8E\xBA\x97\xCE\xC4\xB0\x85 \x0E-\xD5\x8E\xF5\x94\xEB\x8D\xA5ag\x94\x17\x14w2A\xD5\x19\xB3\xF2\xCC8 J\xF2\xB0\xDE$\xECo\xE1K\xAE\x09\x08\xEB\xE5\x17\xFA\xFE\x17\xB8\xB56~'\xD8\x00O\xAE\x1F"M\xCE\xD7Hy\xDE"+X\xFC\x80\xA3\x0E\xAAP}TX4x\x8A\xDA\xD0\x0D8\xB7\xD9\xAA\xDA#\x01\xF0\xB0SI\xDAV\xEB\xF0'\xA1\x12\x8D]%\xCC&\xE8\xBC\xB35\x85\xF7\xC7\\xAA\xD7\xC3B\x9B\x19\xDC\x0B\x7F\xD6b{3.\xE1\xFFU\xEAq/g1\xE6Prik\xAD\xE2\x0D\xD8s\xC6\x03 \xF2\x07\x12+5)\xB1\x0Df)\x02\xE5\xEBB\xB3w\x9F\xD0i\xF0\x18J?\xB5|x!\x9AR9w\xA0\x1B\xE7\xB8n:\xA4\x17Gv\xBB\xC5\xB1\xB4\xC7\xD1\xD9\xAB\x9B\xAF?\xC9q\xFA\xF6;\x0C*\xEA\xDD\xF7\xD1n\xC7%b\x8E\x1A6\x824\x0Cn\x83\xAF?\x91\x82K4\xDA:~5\xBCJO\xB4\xC3\xF5"\x18\xA5Y\x18Q.\x91z\x96\xF9\xAA\xF0\xFEt%\xF5\xD48\xD2C\xB0\x1C](2\xD9\xE3Afw9R\xAF\xB2
\x0Bq\xE8\xBE\xE3\x1A\xB72B\x93\x12\xAE(z\x9E\xEC\xE0\xE1\xA80\xB3\xDC\x18r\xD2+\x02\xD5h~o\x11\xA3M\x0F\x10\x11UH\xB3(\xE6\x86k\xC4\.\xA8\xC29;/`\xA0EV\xA8\xBD\xF0Q\x80a\xCAqs\x09 K7}WO\xE2\x0C\x8B\x0EQh\xF9\xFC\xFD*\x9A\xDB+\xF1\x99\xB7Q\x19\xCB\xED*wX\x14<\xC7\xC4P\xF1\x8C\xB8a2\x87\xC5q\xF7\xF6\xD5j\xEB?\x8AQ\x9E \xFAv\xA5Y\xE1\xC4\xED\xDFP*T^|\xCF\xEE\xF3\x82`\x14d\xCD#\xF3\xE0\x80Hw\xFB\xE5\x03s\xFC\xD1\x80\x02:\xB4rCf\x13\xC6\xD2\xEF\x1C\xAA\x08\xB2\x90\x81\x82.\xAFq\xF1j\x19<\x08i\x19\x0B\xFD9\x8C9\x8CL{D\xD1\xE5\xA2[\x941J\xAE\x0B\xB0\x01o\xBC\x19lAW\x0C\xA5C\x8C\xB8\xA7\xC3Vs\x1A\xFA\xD7\xAD\xA2; p#\xD1?\x97\x04\x9B\xB3\xF9\xB8\x0B\x007y\xBCwu\x8BM\xEE.\xA5\xBCv\xC6r\x9C\x1DmE\xE0\x83\xCD`\x1F\xAD\xA3\x17w\x90\x90\xDCG\xE6\x0A?\xCA\x05u\x18av\xB9=\xF6\x83O\x0C\x18=\xF0\xB9\xE3\x18dG\x82O\xD7\xA87\xD0\x0CY\x05\xD6\xAC\x99ZJ;\xFCV\xD6\x00,p\x03\xC9f.+s\x04\x93\x05"k9\x85\xE2\xAF\xB6&&^\x0C?\xB7Y\x80\xE9\x00\xCB!\xE5\xEED\x08\xB9\x95\x80'\x97b`\x19f\xC3\x81\xD8\xEE\xFCw\xF9M\
x8C\xDE\x0F\x8F\xB3\xC2[@F~\xA6J8\xEA\xA8UY`\x9E\xDB\xB9\xBC\x92\x8E\xAF\xE4'0\x85 \xC7\xD8+\x0F\xC2\x12Y\x0F\xD5\xA5\xB7\xEF$\x9E\xA1\x98\xC7L\x0E\x90]\x8C\xBC;\xEE\x8C\xB4\x12\x95\xBCV\x1D\x8FQ\x19\xED\xF84\xEA\xEB&\x18B`0\xB9e\3k8\x9BK3\\xB2\xA0\x14\x1B-3\x91\x95\xB7\x9F\xDF\xE1\xB8\xE2\xCD7\xF88!\x0B\xF57AYX3\x09m\x93g\x0E[.'>\xE8|K\xFE\x09\x87\xAC\xEC\xEDqJtVH\xCA\xFB\xCD5\x9F\xA9%@\xF5\xF1\xEA\x8E\xD5\xE0\xC4lY\xC4\xA5\xE7}\xE2\x88\x00\xA5\x84\xF7a\xEC\x02\xDE\xE7kC\x97\x8Df0<\xDFKZe$\xFB(\xA7`\x8C\xA9\xB4}q\xF9N\xE3\xDAiS\x93\xDA\xC3\x9FOK\xB9\xF7\xCDc\x97\xEF\x92\xBD\x80v\xF3\xABM\x0A\xAF\x84n\x01\xF2\x14 \xA8\xC8x\x17\xF1U\xF1\xF3\x1B\xF1\xCC\xE7\x8D\xA9\x0A\x9Cy\xCB\xE2\xD0g\x1Bf\x94\x0D49E\xEC\x19o\xB0\xB5\x9CH,\x83\xBB[\x90\xA3\xB3\xF6\xAE59\xC5o\x8F\x8F\x14\xA4\x1E\xC3\x9E\xF2\xAF~e\xE9Zo\xFFV\xA9\x1D\xCA\xF76\x1C(\xA0\xFFm\xAA\x03Dy\xDC\xB2\x8E\xF9j\xF9\x05\x92\x1E\x0C@P\xEC\x085\x90\xAE\x03T`\x99#\xBF\x93\xECk\xABk\xB0k\xF4\xCA\xF0\xB3:\xB5\x0B\xBF\xBE6n\xF0\xA5!\x0Af
\xFF\xBFN\xD8zL%\xB3:\x7F{U+\xEE\x91\x00\x95o\x02\xF6w\x85\xC8\x869\xE1Ri\xB9\x9AW\x1D\xE5c\xAC6\xFE\x1D\xB6\xD3\xAE\x03\xA8A\xCE\xABC\x03\x0D\x11\x0A?r\xC0\x0D\xC0\x9D\xC6Z\xEC\x9F\x1A\x9C\xB9A\x84.P\x81\xE1\xC36_\x87\x1A\xC63J\xA0J\x11<!\xB1\x1F\xCE \xD1\x0B\xA1y\x81\xBD5p\xFF\xA9R\x07\xD9\xEA\xFE\x9C\xF0M55\x8DE\xE8\xB2\x16\xB9\xDF:\xB0!\xAD\xAE\xA9\xD8,\xB50\x1C1\xB0\x9A\x84\x9E\xF4\xC0\xC9\xAA\\x15\x91}\x08\xA2\xEE\x95\xD1\x12-\xD0k\x96\x9Ao\xAA\xA8Z\xAB\xE1\xC8M\x87\xBE\x17\x7F_\x90\x13cr1\x8E\xF9\x91W\x96\xBF\xD7\x87\x90\xD9F\xC3x\xAD\xBA{\xEF\xC0an\x11\xB4h\x96\xD2\x8E/1b\xF0\xB4\xF3\xE0\xBB\xEA\xAC\xB4\xDA&QE\xE4]\xB3~\x03F}|\xB7\x18B\xB9\xCB6\xAE\xC5\xB8\xCCZY`$\x0C\xE4h\xC1~\xB0\xF7n#\x15\xA5\x96\x09\x8E\xE0b\xCB\xA6\xCB\x9E\xA7\x94\x8E\x06\xAAb\x8E\x97\xBEc\xBDK\xAA\x98\x89\x96c\x86\x0A\x90\xC9\xB3\x88TwM\x07pb\xD5\x9DBJ\xFCA%\xDE\xDE(\x18\xCC|8\x03K\xA5\x9E\xADV~\x1D\xE0\x0D'\x13_\xAA\xBF\xEAe\x8D\x8B\xEE[B_\x0E\x90\xA1\x9Dox\xD1\x94\xB8\xFC\x8E*\x19\x8C\x0C\x82:\xA8\xB
4s\xAFe\xF5\xF8qJ\xA4$~\xC11Z\x8Dzp\xA2\xA1\x05\xAC(\xB9;a\xEC\x1F\x7FW9\xD4\xA4F\xFF\x98t\x88\xB1\xE0\xBD\x1A\xD9\x87Rp\xE5\x13\xCA\xDE4m\xB2\x08\xEF4\x96m\x09\x99\x14\x9C\x9E\xFB\x85\xABpr\x829\xB6Mq\xDE\xE6YoHl\xDE\x9AZ \\x944\xBDS9\xE7\xBA\xFD\x9FU\xFD\xD2n\x94\x19\x0D\xE5\xDCyn\xA9\x05\xF7-|C\xDFl\xB9\xBE9\xFEf}\x05\x95#\x9F\xD6\xF2\xB7\xAA\x8C\x1Bz\xB62{\xDB-Ie\xC1\xCD\x1A\xEEc\x86\x13M*\xB5\x14\x07\x06"\x91:\xA9j\x0D~\xB7\x11\x18o4\xD5t\xC6\x04G\xE3\xAD\xA5\xE9\xF9\x03\x12>\x01J\xA3\xE9\x12\x87\xF6\xD2W\xC9\xF3\xD9\x1F\xF5>\x05h\xB3\x1D\x9D\xD8\x8B\xE2\xAAx:X \x0C\xA9\xF8\x1B\xB3v\x0F\x0C7\x83\xDC\xC2\x96\x90{>\x83q\xB0\xC8\\x99)N\x87L`\xDCRQo;\xF8o\x8C\xE2\xF8\xD3\x0C\xA3er\x02G\x89I\x91\x011\xA8\xB2<\xED\x85B6F{H\x87\xA6\xC1\x93Er\xE8\xC6\x01/`n\xAE\xFD\x91\xD6\x07e\x0C\xD9J\x9A\x82\xB1\xC3{\x17\xC4\xFB\xE6,\xCC8R\x94\x03\x193P\xA4C\xAD8\xBA\x19\xF3\x96k\xC1{\x09\xCA\xF6\xF9\xBA\63\x96\xFF\xB7\x86\xD9\xFE\xA7\x8Bs\x0F>\xE3<\xA7\x88\xAE\x14\xAC\xFE\xD4\xFB\xF7\x93\xA4\xAF1\x
DD~\x0B&\xE8]\x02\xB9\xE1\xA5\xCBW\x8E\xBD\xA2\x9E\x19\x13\x0CS\x94\xC3\xC9\xB1\xB7i\x0FS\xE4N+\xACg\x085\xC5\xBB\xE8\x1F\xA9\x07c\xB14\x10\x18\x86\xA9k\x9A\xAF\xB2\xE9"\xBAn\xC2\xCA\xF2~\xCC\xE0s0xo?\xB6G\x84\xD2\xFDZbCT\xB5\x95g\xBE\x1D\xF9\x1D\xD2\xB7g\x057r*\xE9\x1A\x1CCo\x84\x9F\x95\xDA\xB6\x0A\x16Z\xD5\x09,\x1C\\xF1v]~a\x93GB\xD6\x91\x14\x99r\xAD\xB6\xED\xE8\xBC\x0D\xC3\xA3\xB4\xB4J\xB0k\x83\xC5i\x0B\xAAS<v\xF4\x1F\x1F\xE3\xF2\x99\xD1X\xA2\xD0\x19\x02Ob\xAF\x80\xA2\xDEV\xA8\xEF8eb\xADW\xF2\xB8\x9C&\xBB6\x86\xEA\xB3\xF8J\xF0\x07$\xB1Q\x8B\x9F\xBF\x06\x03\x8D'\x18\xBD,\x9C\x0C<\xF4\x96\x87\x07YO%x\xAF\xF2\xBE\x00\xE5<b\xD59\xF8B\xAA\xD6\x14\x0B\xCF\xC2\x8D\x88\x83JG\x94\xA1\xDDFE\x94\x99v\x84+\x95\xC9\x8B\xC8\xE6\xF9\x93\xD0\xF71\xFD\xDB\x1E\xF4\xC8&S\xB4\x9A\xC6k3d\xE1\xCC\x01\xF6t\xF8\xC9\xDB\x93\x04\xCF\xD8^u\x15&\x17&]\x91\x9AH\xC96\xA6\xE6$\x8E(\xD3\xCBS\xBAM0\xF4\xB1|v\xA7^\xEF,>\xA6\x82\xBFF\xAA\xB7sA\xD7\x7F\xAFub <\xF87o>\xFC\x02\xE3I\x09\x8Al\xE8\xC6\xA0\x0D\x91\xEB^\x
8E\x8C\x1E\xDB\x89Q\x03-H\xB2Ej^\xF8>&D\xED>^\x00W\xF8L\xCA\xE0\xFF\xDEU\xED \xC5d\x16a\xC7\xF4\x85\x01\xE3\xDC\x9D\x8D\xD6\xD9B\x1E\xCE\xE5\x95\xBD\xFF\xD5_\xF4\x05\xA2\xDAx\xA6=\x8F\xB3\x7F\xA2\x81\xC3\xE5>\x93\x1B\xBF\x9EY\xD8j\x88)%\x0A\xBC\xA2_\x80 *'\xF6q{>\xB5D\xC8\x8E\xDEW\x9D}\xA2E\xAFd\xA8y\xB2\xD3\xC5a*8[\x0B\x17\xCF\xC2n\xFC-\xDF7)\xC1\xB2\x8Cp.ie\xE3\xCD\x0B\xEDp\xA1\x19l\x8E\xD1!\xEAb\x0F\xDF\xFB\xEB\xFD\xE7\x8F\xB9\x96\x1D\xED\xE7Tx\x7F"%\xBDk\x19t\x8E(\xF6\xCDc\x9A\xBDG\xA5$1\xB5\x96\x18\x82\x9BS\x1B\x8Bg\x93{O\xB1\xE9\xC5,\x98\xBCh\xB36\xC2\xAE\xF6\x98\xC0\x97\x00PCO\x95\x0A\xFD\x00\x00\xA2\xC0D~\x9Abu\xC2t]\x11\xDE\xC1\x0C\b]F\xAA\xE7\xE34\xCFf),\x0AnA\xEDF\x14\xCBen\xEA\x99w\x19\x0D\x9B\x95y\xFCn\xE4\x8Cj7\xAE\xF9\xCCEC6\xDB\xB6\xCD\xCCzb\xD9\xB2\x95\x17*+5\xC7\xE3n;\x0CHq\xF0\x80*Kv\x93}\xD0\x94\x89n\xB3\x90\xFCc\xCA\xEA\xC9Ph\xEAH\xE9\xA2\x8B\xF7\x07\x81\xC4\x0F7\x93in\xE4E\x9BH5\xCB\xE2.\x8C\xE2\x02\xA5\x19\xA8 \xA5\xD6\xA4\xF3e\xC8\x8Au\xF3\x8E\x06 \xD0\xD9\xC
9\x09\xBF\xF4\xD01\xAFo\xC8\xFF\xE6\xE6 k\x7F\xB8\xA6=z\x08\x13y:\x9A#\xBA\xF67\xFDz\xDE\x94\x93\xF1\x93)\xB4\xEE$\x0D\x19<\x171\x0B\xDD\xFD\xE9!\xF9\x9F\x14\xEC\xB3\xD5S\xF0\xEB2q\x04\x87\xAF\x82e\xDE\x1B\x9B]i\x88p\xD9q]G\xD7\xB8\xB4\x07\x87C\xC1\x80a\xA7\x0B\x91\x16\x14L\xC2\x80O\x1A\xCCn\xEA\x1DZ\x91\xD9\xC2\xCB\x85}\x06\xC33\xC5\xDF{%6^@\x09i\xF1(\x9Bb\xDB\x06\xD2\x96\xDB\x90>\xE3qn\xF0`9|\x1Bs\x94\\x97\x0D\xDC\xB1z\xDBK\x1BFYc\xD2\xEC|\xECx&}\x12`\x0E\x9B\x86%\xF1\xA5\wY\x0E?\x1E*\x95\xAD\x96\xDBm/d\xE6\x90\x80\xD8n\x00\x81\x8B\xB4\x19\xC4@\xB3\x8E\xDD\xB0"\xDF\x1A\xC7\xF1\pf\xB2\x09x\xA9\x9D\xC4\x06\xFFrv+\xDF\xF5\x12\x0D\x95e>YJw\xB8\x8D\x14\xD6\xAEj\x16\xBE\xE9H\xBA/\xE5\x08\x9C!\xCA\x9E\xB7\xE8\x82\x8C\xB7hg\xF9I;\xD5\x1C\xCD]n\xBEe\x86X\xC5MI\xC0\x17h\x0D\xD8I\x83a\xFF\x1D\xE8\xD5\xBA\x8DD\xF7\xF0#\x07\xFA%\xBF\x0A\x94\xB8.Y\x9A\xEA\xA7?@x\xE5\xF9\x16\xE9\xCF\x97\xE4Cyo\x16F^o\x99\x9BZ\xEC\xE1\xD9\xD4~\x10\xA3\xAF!Y\xF0\x11e\x9B\x18\xECw\x98sP\x86:S\xFE\xF7\xA7\xDD\xF2+\x
C8\xC4\xE3\xC6\x80\xC2U\xB7\xA2\xAB\xA6\x9E,!\x0A\xAA\x93\x0A\x1B{T\x0F%P`\xAB\xCC\x8DP|\x80\x8D\xA2\xDB\xF7\xCB\xE6?\x1C\xBD\xF5\x93\x8E\xDA\x8AK\x19J\xB4\x84\x95\x0F:y\xCE\xBB\xB0\xD9W\x85\x11\xDC\x02\xEB\xF0\x83B#\xC8\xA1l\x0CG\x8Ds\xD9 k\xCD\xBC\x9B\xCE0g\x04\xC7kUB~\xE5\xB1\xA0\xCF\xB9\x13\xB8\xD3fAU\x06f\x1D\xD4\xE9x\xB7\x18s6\xB2=~\xF7[.\x8BC\xFF\xD4\xAF\x13\xECat\xA7\x93N\xC7v\x12\xD4\x08\x95|@>Q\x17\x95\xD6i\x1F\xB8W\xA8\xC2\xE3\x11A\x12\xB1\xCCQ\xE8\x9A\x8E\x13\x08&\xD4{\xF4C\x09csv#\xCA\xE0\xBBB\xFFZ\x0D\xC97\x8D\xDA\x8Ar\x0E\x1A/a]\xDB\x1E\x09\x84\xBA\x96\xF3\x81\x96\xF5\xA1@\xC4\x97>O\xEAN\x96IU\x13I\xBD=\xC7\x88\xDB6\xAE\xDCz\xD3d\xD7\x04Z\xF4M,GD\x03\xE17T P\xEC O\x9A7\x13\xBA&\x14.Z\x99\x91N\xAD\xAF\x12\x11\x92\xF6\xB5\xADn\xA6[?\xA1g{Iv1Q$N\x13\xC1B\x18\x02\xCD\xACb\x00\x91\x07\x93\xC0h\\x8F\xC8pK/\x1F\x17\xD6x\xE6\xB3\xDDX\xA07\xFD>\xEE\xE5\xFE%O\xDC\xB4=R\x8D\x0D\xA4.\xE3\x18\xEA\x91\x19\x82\xDCK\xEA\xEA\xE7\xE6$sv\xF7*\xD0\xCADCSX\x07\x91\xCA/\xDAb\xA2\xC5\xED\xA
0\x95\x02\xC6G\xA7\xB6Bjz3r\x9A5\x90\xDD\xEB\xD1%\x8D\xF1\x8E] \x8F\xC4\x9C\xFD\xA3\x082k\xF6[\xFB\xF8\xC8\x95r%X\x07\x03\xAA\xDB*\xFB3>\xD3\x83&z\xAD\x91^\x97\x9E\x90\xDC\xCC\xD7\xFD\xE3\x9Ba:\xB4\x12k\xB9\x9B\x03o\xE4\xF0\xA7\x08#U\x16&\x9D\xF5A/\x87\xF7\x86\xAD8\x82\x1A\x9A\x86\x0B\xE0H\xEDZj\xDC\xA6\x1D\x92\xA8\xC6\xA2\x00a\x03\xA3\xBF\xCA2\x11\xFBT\x85\x04\xB9\x15g\xA5\xE7N\x08\x02\x19\xAD\x81\x1Eu~K\x0E\xF5\xF8}]\xC1K\xAEe\xD6L\x80~\xEF\xD1\x07\xC53\xA0\x12\xF9\xB0&\x0D\x9B\xCDF*\x03">,\xA6\x98\xD4A\xD9\xC8\xD2\xCA\xF9.\x19\xCF\xFD6\xF1;\xB1Kn\xD4\x0C#\x90\xF4\xF3P\xE8^\xF4\x1Fv\xC1\xAD\x15\xFF.Q4\x0C\x10O\x04_\xE6\xE6)\xDA\x84\\xA6\x96k\xBD\xD3wrZ\xCA\xF0Z\xE9Q\x80\xF9\x07px\xC5{o\xFC\xCE\xB6\x07\xC0\x94\x15\xA3\x18\xF8'\xC7i95\x89n\x0D\x83\xE3\xD8\x9C\x19\xF3^\xB2\x0C\xACni\x13\x1B\x10d\x02,,)\xE6\xE2\xCD\xE2\xC4\x8F\xF7\xC4|0D\xA3\x0F\x97\xE9gA]\x1DB\xB3\xC4\xB9\xE3\xEE\x0B<\xCA\xF1EG\xFAA%\xF7@\x87^r6\x9B\xE5\x87\xE2e\xD9\x02B\xF6\xA2\xD4\xF1\x8CL_\xDF\xA9\xE4\x97\x09\xF2\
xDB\x9F\xE0\xBB<\x07E\x88\xB47\xDDF\xAFG\xCD\xD5\x15\x95y\x0Bp\xD4|\x8FC:m\xDD\x0B\x96\xBD+\xBF\xCF\xB9\xBC\xC4\xA8\xD8\x98\x8D\x99\xAC%\xD0\=\x10\xBA\x95\x9A\x06\xE2L\x94\xAF\xEB*\x80\xE8\x00\xD2\x9A\xE1GX\xE6\xBF\xBC\xDE\x14\x04\xD4\xFE\xB9\x9D\xCF\x8B^\xEDID\xFA\xA3J\xFA]K\xA6!rHQ\xCC\xE4\x1EO\xAD\xCC\x9F\x9E\xE1\xC6!\x9A\x94\xBD5\x9C\xC1.\x83\xB5\x0A.k\x0E\xC0\xF0d$\xEF~\x81\xB0\xD1\xF5\xB5?kS\xF2A\x07\xF2\xB8\x03\x95\x97\x85i\xC1\x01\x8C\xEB\xA0\x92S\x11\xD4\xD9\xACs\x03E\x97\xF4\xB8\xBB\x8E\xD8%ju\x1E\xBCO2\x09\x1C\xC9\xB6+\xA9\xFF\x1C\xE2\xE0\xC5xc\x18\x99\\xB2,J\x15\xF9\x197Ez\x96\x81\x8B\xBF\x14\xC6\x9A~\x9D\xF9\x94\xD2\xCC\xA4\xAC\x8Fe+\xDF\x1F\x85%\xE1s\xF5v\xA7\xAA\x1D\xFF\x17\x1A2\x0D\xC8z\xC6\xF7u\x08Sr.\xA6\x8A@i\x8E$jH\xAF\x10eU z\x94B\xBD\x0D\xBC&4X\x07\x8C\xBBf\xB3`\xE8c\xA0\x04g\xE5u\xDFc\xAF\xED\x17\xE1X\xB6\xCC\xD8k\xD6\xDC\xCEX\xAD\x00W]m\xB2u\xA8\x83\x83\xCB\x1B\x97\xCC\x13\xE3\xBC\xF0\xA6\xAEdG\x8Ckhm\xAA~N\xB6\xF8\xD5#\x83"\xAD\xC0L\x91\xEF \xD5\x04A\x8D\xBF
v\xE9\xE0\x19~vBk\xAAqA[\x1A\xB0v1\x11\xB1\x03\x08\xD4 \x85oa\xFA\xF9,\xDD}\{\xDC\x93\x80\x090I>\xC7\x1E\xAC\xF9qT\xAA\xA9; \x8FG\xE4\xB9p\xC4\x0Ex\x91\xB0%\xE3\xDB\x01s\x96\xC7\x92\x86\xC0\xF9\xD3r\x9B%\xBC\xCF\x86\x07\xA7E\xC8(}6(X\x8C\xEAh&\x9D\xCFd\x83 \xEF*\x1F\xDB\xA6\xD3%\xBF\x8DV\xB9\xAB/4`\x15\xCC\xA5\xCEF\x05\xC9\x9Cm\x8BM\x0AI3}L}\x9ET\x1EwUQ+\x88\x84\x81\x92[\x14\x87\xDE\xFBo\xE7\x09\xD8\xADa\xAB\xFCN\x1Cm\xF5\xA3_2\x84\xB63\x03W)\x83S\xF2d\xB4\x14T\xC2\xEF\x87D\x9F\xA0\xD1}\xA2g\xE5R\x84$\xA7\x11\xE2\xCAd\xC2\xA3Rs\xBE\x8B\xA6j\x8AB\xBF1\x93\x0A\x97\x8DA\xCD\x98\x9B\x12y\xFE\x0B\x8E\x98Af\xC5\x0D\xCE\x86iJ`\xB5\xC4.- \x1F\xAB\x93\x8D\x08\xE5o\xBB&;\x99\xE5\x89I9nNS\x978kiI\xA5\xD0\xD6\xC1\xA2\xD3\xD3%\x19C+8u\x19\xDB\xF1\x8C\x8CP{\xB2g\\xB9\xA1:#\xE9\x96\x12\xEC\xD4\x92\xF1\x1D^\xFE-b\x87\x93\xCE\xE2\xF7\x99\x1DB\xC6\x85\xEE\x83y\xDCF)\x04\x96zz\xDE\x00\xFD\xA3\xF6\xA1\xEA\x8DDhcMX\xD6[)@\xB6\xE2E\xF2+,\xD3\x84\x0CT\xA5*\xC4:;\xC6\xF5"\xE9\xD2)|\xAE\x11\xB8\xDFK\xECo\xC
AB\xDE
<TRUNCATED>
[34/50] [abbrv] git commit: MAPREDUCE-6067. native-task: fix some
counter issues (Binglin Chang)
Posted by ar...@apache.org.
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00322161
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00322161
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00322161
Branch: refs/heads/HDFS-6581
Commit: 00322161b5d4d54770c2f0823e036537edecf5bf
Parents: 1081d9c
Author: Binglin Chang <bc...@apache.org>
Authored: Fri Sep 5 14:20:39 2014 +0800
Committer: Binglin Chang <bc...@apache.org>
Committed: Fri Sep 5 14:20:39 2014 +0800
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../NativeMapOutputCollectorDelegator.java | 4 +
.../mapred/nativetask/StatusReportChecker.java | 5 --
.../src/handler/MCollectorOutputHandler.cc | 3 +-
.../src/main/native/src/lib/IFile.cc | 6 +-
.../src/main/native/src/lib/IFile.h | 3 +-
.../main/native/src/lib/MapOutputCollector.cc | 85 ++++++++++++++------
.../main/native/src/lib/MapOutputCollector.h | 8 +-
.../src/main/native/src/lib/Merge.cc | 24 ------
.../src/main/native/src/lib/PartitionBucket.h | 2 -
.../src/main/native/src/lib/TaskCounters.cc | 10 +--
.../src/main/native/src/lib/TaskCounters.h | 8 --
.../nativetask/combinertest/CombinerTest.java | 8 +-
.../combinertest/LargeKVCombinerTest.java | 6 +-
.../nativetask/compresstest/CompressTest.java | 3 +
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 74 +++++++----------
.../mapred/nativetask/kvtest/LargeKVTest.java | 82 +++++++++----------
.../nativetask/nonsorttest/NonSortTest.java | 6 +-
.../nativetask/testutil/ResultVerifier.java | 24 +++++-
19 files changed, 179 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 269a2f6..279b960 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -20,3 +20,4 @@ MAPREDUCE-6054. native-task: Speed up tests (todd)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
+MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 224b95b..828d7df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.util.QuickSort;
/**
@@ -46,6 +47,7 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
private JobConf job;
private NativeCollectorOnlyHandler<K, V> handler;
+ private Context context;
private StatusReportChecker updater;
@Override
@@ -58,6 +60,7 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
handler.close();
if (null != updater) {
updater.stop();
+ NativeRuntime.reportStatus(context.getReporter());
}
}
@@ -69,6 +72,7 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
@SuppressWarnings("unchecked")
@Override
public void init(Context context) throws IOException, ClassNotFoundException {
+ this.context = context;
this.job = context.getJobConf();
Platforms.init(job);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
index f152074..1e76d39 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
@@ -76,12 +76,7 @@ class StatusReportChecker implements Runnable {
reporter.getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES);
reporter.getCounter(TaskCounter.COMBINE_INPUT_RECORDS);
reporter.getCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
- reporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);
- reporter.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS);
- reporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
reporter.getCounter(TaskCounter.SPILLED_RECORDS);
- reporter.getCounter(TaskCounter.MAP_OUTPUT_BYTES);
- reporter.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
}
public synchronized void start() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
index 7967034..4df5e64 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
@@ -18,6 +18,7 @@
#include "commons.h"
#include "util/StringUtil.h"
+#include "lib/TaskCounters.h"
#include "MCollectorOutputHandler.h"
#include "NativeObjectFactory.h"
#include "MapOutputCollector.h"
@@ -94,4 +95,4 @@ KVBuffer * MCollectorOutputHandler::allocateKVBuffer(uint32_t partitionId, uint3
return dest;
}
-} //namespace
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
index d11d823..4a5edda 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
@@ -98,7 +98,7 @@ IFileWriter * IFileWriter::create(const std::string & filepath, const MapOutputS
IFileWriter::IFileWriter(OutputStream * stream, ChecksumType checksumType, KeyValueType ktype,
KeyValueType vtype, const string & codec, Counter * counter, bool deleteTargetStream)
: _stream(stream), _dest(NULL), _checksumType(checksumType), _kType(ktype), _vType(vtype),
- _codec(codec), _recordCounter(counter), _deleteTargetStream(deleteTargetStream) {
+ _codec(codec), _recordCounter(counter), _recordCount(0), _deleteTargetStream(deleteTargetStream) {
_dest = new ChecksumOutputStream(_stream, _checksumType);
_appendBuffer.init(128 * 1024, _dest, _codec);
}
@@ -184,6 +184,7 @@ void IFileWriter::write(const char * key, uint32_t keyLen, const char * value, u
if (NULL != _recordCounter) {
_recordCounter->increase();
}
+ _recordCount++;
switch (_vType) {
case TextType:
@@ -214,7 +215,7 @@ SingleSpillInfo * IFileWriter::getSpillInfo() {
_codec);
}
-void IFileWriter::getStatistics(uint64_t & offset, uint64_t & realOffset) {
+void IFileWriter::getStatistics(uint64_t & offset, uint64_t & realOffset, uint64_t & recordCount) {
if (_spillFileSegments.size() > 0) {
offset = _spillFileSegments[_spillFileSegments.size() - 1].uncompressedEndOffset;
realOffset = _spillFileSegments[_spillFileSegments.size() - 1].realEndOffset;
@@ -222,6 +223,7 @@ void IFileWriter::getStatistics(uint64_t & offset, uint64_t & realOffset) {
offset = 0;
realOffset = 0;
}
+ recordCount = _recordCount;
}
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
index 76d6fbc..d86e98a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
@@ -129,6 +129,7 @@ protected:
AppendBuffer _appendBuffer;
vector<IFileSegment> _spillFileSegments;
Counter * _recordCounter;
+ uint64_t _recordCount;
bool _deleteTargetStream;
@@ -153,7 +154,7 @@ public:
SingleSpillInfo * getSpillInfo();
- void getStatistics(uint64_t & offset, uint64_t & realOffset);
+ void getStatistics(uint64_t & offset, uint64_t & realOffset, uint64_t & recordCount);
virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) {
write((const char*)key, keyLen, (const char*)value, valueLen);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
index d2b116d..d151f1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
@@ -76,9 +76,11 @@ void CombineRunnerWrapper::combine(CombineContext type, KVIterator * kvIterator,
/////////////////////////////////////////////////////////////////
MapOutputCollector::MapOutputCollector(uint32_t numberPartitions, SpillOutputService * spillService)
- : _config(NULL), _numPartitions(numberPartitions), _buckets(NULL), _keyComparator(NULL),
- _combineRunner(NULL), _spilledRecords(NULL), _spillOutput(spillService),
- _defaultBlockSize(0), _pool(NULL) {
+ : _config(NULL), _numPartitions(numberPartitions), _buckets(NULL),
+ _keyComparator(NULL), _combineRunner(NULL),
+ _mapOutputRecords(NULL), _mapOutputBytes(NULL),
+ _mapOutputMaterializedBytes(NULL), _spilledRecords(NULL),
+ _spillOutput(spillService), _defaultBlockSize(0), _pool(NULL) {
_pool = new MemoryPool();
}
@@ -108,7 +110,7 @@ MapOutputCollector::~MapOutputCollector() {
}
void MapOutputCollector::init(uint32_t defaultBlockSize, uint32_t memoryCapacity,
- ComparatorPtr keyComparator, Counter * spilledRecords, ICombineRunner * combiner) {
+ ComparatorPtr keyComparator, ICombineRunner * combiner) {
this->_combineRunner = combiner;
@@ -128,7 +130,15 @@ void MapOutputCollector::init(uint32_t defaultBlockSize, uint32_t memoryCapacity
_buckets[partitionId] = pb;
}
- _spilledRecords = spilledRecords;
+ _mapOutputRecords = NativeObjectFactory::GetCounter(
+ TaskCounters::TASK_COUNTER_GROUP, TaskCounters::MAP_OUTPUT_RECORDS);
+ _mapOutputBytes = NativeObjectFactory::GetCounter(
+ TaskCounters::TASK_COUNTER_GROUP, TaskCounters::MAP_OUTPUT_BYTES);
+ _mapOutputMaterializedBytes = NativeObjectFactory::GetCounter(
+ TaskCounters::TASK_COUNTER_GROUP,
+ TaskCounters::MAP_OUTPUT_MATERIALIZED_BYTES);
+ _spilledRecords = NativeObjectFactory::GetCounter(
+ TaskCounters::TASK_COUNTER_GROUP, TaskCounters::SPILLED_RECORDS);
_collectTimer.reset();
}
@@ -155,9 +165,6 @@ void MapOutputCollector::configure(Config * config) {
ComparatorPtr comparator = getComparator(config, _spec);
- Counter * spilledRecord = NativeObjectFactory::GetCounter(TaskCounters::TASK_COUNTER_GROUP,
- TaskCounters::SPILLED_RECORDS);
-
ICombineRunner * combiner = NULL;
if (NULL != config->get(NATIVE_COMBINER)
// config name for old api and new api
@@ -166,7 +173,7 @@ void MapOutputCollector::configure(Config * config) {
combiner = new CombineRunnerWrapper(config, _spillOutput);
}
- init(defaultBlockSize, capacity, comparator, spilledRecord, combiner);
+ init(defaultBlockSize, capacity, comparator, combiner);
}
KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t kvlength) {
@@ -182,7 +189,7 @@ KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t k
if (NULL == spillpath || spillpath->length() == 0) {
THROW_EXCEPTION(IOException, "Illegal(empty) spill files path");
} else {
- middleSpill(*spillpath, "");
+ middleSpill(*spillpath, "", false);
delete spillpath;
}
@@ -193,6 +200,8 @@ KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t k
THROW_EXCEPTION(OutOfMemoryException, "key/value pair larger than io.sort.mb");
}
}
+ _mapOutputRecords->increase();
+ _mapOutputBytes->increase(kvlength - KVBuffer::headerLength());
return dest;
}
@@ -272,10 +281,9 @@ void MapOutputCollector::sortPartitions(SortOrder orderType, SortAlgorithm sortT
}
void MapOutputCollector::middleSpill(const std::string & spillOutput,
- const std::string & indexFilePath) {
+ const std::string & indexFilePath, bool final) {
uint64_t collecttime = _collectTimer.now() - _collectTimer.last();
- const uint64_t M = 1000000; //million
if (spillOutput.empty()) {
THROW_EXCEPTION(IOException, "MapOutputCollector: Spill file path empty");
@@ -293,10 +301,24 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
info->path = spillOutput;
uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime;
- LOG(
- "[MapOutputCollector::mid_spill] Sort and spill: {spilled file path: %s, id: %d, collect: %"PRIu64" ms, sort: %"PRIu64" ms, spill: %"PRIu64" ms, records: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64"}",
- info->path.c_str(), _spillInfos.getSpillCount(), collecttime / M, metrics.sortTime / M, spillTime / M,
- metrics.recordCount, info->getEndPosition(), info->getRealEndPosition());
+ const uint64_t M = 1000000; //million
+ LOG("%s-spill: { id: %d, collect: %"PRIu64" ms, "
+ "in-memory sort: %"PRIu64" ms, in-memory records: %"PRIu64", "
+ "merge&spill: %"PRIu64" ms, uncompressed size: %"PRIu64", "
+ "real size: %"PRIu64" path: %s }",
+ final ? "Final" : "Mid",
+ _spillInfos.getSpillCount(),
+ collecttime / M,
+ metrics.sortTime / M,
+ metrics.recordCount,
+ spillTime / M,
+ info->getEndPosition(),
+ info->getRealEndPosition(),
+ spillOutput.c_str());
+
+ if (final) {
+ _mapOutputMaterializedBytes->increase(info->getRealEndPosition());
+ }
if (indexFilePath.length() > 0) {
info->writeSpillInfo(indexFilePath);
@@ -320,11 +342,8 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
void MapOutputCollector::finalSpill(const std::string & filepath,
const std::string & idx_file_path) {
- const uint64_t M = 1000000; //million
- LOG("[MapOutputCollector::final_merge_and_spill] Spilling file path: %s", filepath.c_str());
-
if (_spillInfos.getSpillCount() == 0) {
- middleSpill(filepath, idx_file_path);
+ middleSpill(filepath, idx_file_path, true);
return;
}
@@ -339,16 +358,32 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
SortMetrics metrics;
sortPartitions(_spec.sortOrder, _spec.sortAlgorithm, NULL, metrics);
- LOG("[MapOutputCollector::mid_spill] Sort final in memory kvs: {sort: %"PRIu64" ms, records: %"PRIu64"}",
- metrics.sortTime / M, metrics.recordCount);
merger->addMergeEntry(new MemoryMergeEntry(_buckets, _numPartitions));
Timer timer;
merger->merge();
- LOG(
- "[MapOutputCollector::final_merge_and_spill] Merge and Spill:{spilled file id: %d, merge and spill time: %"PRIu64" ms}",
- _spillInfos.getSpillCount(), (timer.now() - timer.last()) / M);
+
+ uint64_t outputSize;
+ uint64_t realOutputSize;
+ uint64_t recordCount;
+ writer->getStatistics(outputSize, realOutputSize, recordCount);
+
+ const uint64_t M = 1000000; //million
+ LOG("Final-merge-spill: { id: %d, in-memory sort: %"PRIu64" ms, "
+ "in-memory records: %"PRIu64", merge&spill: %"PRIu64" ms, "
+ "records: %"PRIu64", uncompressed size: %"PRIu64", "
+ "real size: %"PRIu64" path: %s }",
+ _spillInfos.getSpillCount(),
+ metrics.sortTime / M,
+ metrics.recordCount,
+ (timer.now() - timer.last()) / M,
+ recordCount,
+ outputSize,
+ realOutputSize,
+ filepath.c_str());
+
+ _mapOutputMaterializedBytes->increase(realOutputSize);
delete merger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
index b7150ac..474b8b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
@@ -85,7 +85,11 @@ private:
ICombineRunner * _combineRunner;
+ Counter * _mapOutputRecords;
+ Counter * _mapOutputBytes;
+ Counter * _mapOutputMaterializedBytes;
Counter * _spilledRecords;
+
SpillOutputService * _spillOutput;
uint32_t _defaultBlockSize;
@@ -118,7 +122,7 @@ public:
private:
void init(uint32_t maxBlockSize, uint32_t memory_capacity, ComparatorPtr keyComparator,
- Counter * spilledRecord, ICombineRunner * combiner);
+ ICombineRunner * combiner);
void reset();
@@ -149,7 +153,7 @@ private:
* normal spill use options in _config
* @param filepaths: spill file path
*/
- void middleSpill(const std::string & spillOutput, const std::string & indexFilePath);
+ void middleSpill(const std::string & spillOutput, const std::string & indexFilePath, bool final);
/**
* final merge and/or spill use options in _config, and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
index 5434980..126b82d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
@@ -131,7 +131,6 @@ bool Merger::next(Buffer & key, Buffer & value) {
}
void Merger::merge() {
- Timer timer;
uint64_t total_record = 0;
_heap.reserve(_entries.size());
MergeEntryPtr * base = &(_heap[0]);
@@ -153,29 +152,6 @@ void Merger::merge() {
}
endPartition();
}
-
- uint64_t interval = (timer.now() - timer.last());
- uint64_t M = 1000000; //1 million
-
- uint64_t output_size;
- uint64_t real_output_size;
- _writer->getStatistics(output_size, real_output_size);
-
- if (total_record != 0) {
- LOG("[Merge] Merged segment#: %lu, record#: %"PRIu64", avg record size: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
- _entries.size(),
- total_record,
- output_size / (total_record),
- output_size,
- real_output_size,
- interval / M);
- } else {
- LOG("[Merge] Merged segments#, %lu, uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
- _entries.size(),
- output_size,
- real_output_size,
- interval / M);
- }
}
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
index cf5e33b..3be52ff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
@@ -117,8 +117,6 @@ public:
memBlock = new MemoryBlock(buff, allocated);
_memBlocks.push_back(memBlock);
return memBlock->allocateKVBuffer(kvLength);
- } else {
- LOG("MemoryPool is full, fail to allocate new MemBlock, block size: %d, kv length: %d", expect, kvLength);
}
}
return NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
index 7aa7db8..39a7c43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
@@ -22,22 +22,14 @@ namespace NativeTask {
#define DEFINE_COUNTER(name) const char * TaskCounters::name = #name;
-const char * TaskCounters::TASK_COUNTER_GROUP = "org.apache.hadoop.mapred.Task$Counter";
+const char * TaskCounters::TASK_COUNTER_GROUP = "org.apache.hadoop.mapreduce.TaskCounter";
DEFINE_COUNTER(MAP_INPUT_RECORDS)
DEFINE_COUNTER(MAP_OUTPUT_RECORDS)
-DEFINE_COUNTER(MAP_SKIPPED_RECORDS)
-DEFINE_COUNTER(MAP_INPUT_BYTES)
DEFINE_COUNTER(MAP_OUTPUT_BYTES)
DEFINE_COUNTER(MAP_OUTPUT_MATERIALIZED_BYTES)
DEFINE_COUNTER(COMBINE_INPUT_RECORDS)
DEFINE_COUNTER(COMBINE_OUTPUT_RECORDS)
-DEFINE_COUNTER(REDUCE_INPUT_GROUPS)
-DEFINE_COUNTER(REDUCE_SHUFFLE_BYTES)
-DEFINE_COUNTER(REDUCE_INPUT_RECORDS)
-DEFINE_COUNTER(REDUCE_OUTPUT_RECORDS)
-DEFINE_COUNTER(REDUCE_SKIPPED_GROUPS)
-DEFINE_COUNTER(REDUCE_SKIPPED_RECORDS)
DEFINE_COUNTER(SPILLED_RECORDS)
const char * TaskCounters::FILESYSTEM_COUNTER_GROUP = "FileSystemCounters";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.h
index 6afc207..23cedf9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.h
@@ -27,18 +27,10 @@ public:
static const char * MAP_INPUT_RECORDS;
static const char * MAP_OUTPUT_RECORDS;
- static const char * MAP_SKIPPED_RECORDS;
- static const char * MAP_INPUT_BYTES;
static const char * MAP_OUTPUT_BYTES;
static const char * MAP_OUTPUT_MATERIALIZED_BYTES;
static const char * COMBINE_INPUT_RECORDS;
static const char * COMBINE_OUTPUT_RECORDS;
- static const char * REDUCE_INPUT_GROUPS;
- static const char * REDUCE_SHUFFLE_BYTES;
- static const char * REDUCE_INPUT_RECORDS;
- static const char * REDUCE_OUTPUT_RECORDS;
- static const char * REDUCE_SKIPPED_GROUPS;
- static const char * REDUCE_SKIPPED_RECORDS;
static const char * SPILLED_RECORDS;
static const char * FILESYSTEM_COUNTER_GROUP;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index d7f05be..61b4356 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -63,15 +63,9 @@ public class CombinerTest {
final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
assertTrue(nativejob.waitForCompletion(true));
-
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
assertTrue(normaljob.waitForCompletion(true));
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
- assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
- nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+ ResultVerifier.verifyCounters(normaljob, nativejob, true);
}
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index 4ba4550..d054793 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
@@ -87,10 +88,8 @@ public class LargeKVCombinerTest {
final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
assertTrue(nativejob.waitForCompletion(true));
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
assertTrue(normaljob.waitForCompletion(true));
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
@@ -98,8 +97,7 @@ public class LargeKVCombinerTest {
+ ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
assertEquals(reason, true, compareRet);
-// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
-// nativeReduceGroups.getValue(), normalReduceGroups.getValue());
+ ResultVerifier.verifyCounters(normaljob, nativejob, true);
}
fs.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index 3a03d29..b8f9dfc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -69,6 +69,7 @@ public class CompressTest {
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ ResultVerifier.verifyCounters(hadoopjob, job);
}
@Test
@@ -91,6 +92,7 @@ public class CompressTest {
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ ResultVerifier.verifyCounters(hadoopjob, job);
}
@Test
@@ -112,6 +114,7 @@ public class CompressTest {
assertTrue(hadoopJob.waitForCompletion(true));
final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ ResultVerifier.verifyCounters(hadoopJob, nativeJob);
}
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 784c14f..6b658ac 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -114,21 +114,34 @@ public class KVTest {
@Test
public void testKVCompability() throws Exception {
- final String nativeoutput = this.runNativeTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final String normaloutput = this.runNormalTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
- final String input = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- if(compareRet){
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(nativeoutput), true);
- fs.delete(new Path(normaloutput), true);
- fs.delete(new Path(input), true);
- fs.close();
- }
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ final FileSystem fs = FileSystem.get(nativekvtestconf);
+ final String jobName = "Test:" + keyclass.getSimpleName() + "--"
+ + valueclass.getSimpleName();
+ final String inputPath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ final String nativeOutputPath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR
+ + "/" + keyclass.getName() + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ fs.delete(new Path(nativeOutputPath), true);
+ nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
+ final KVJob nativeJob = new KVJob(jobName, nativekvtestconf, keyclass,
+ valueclass, inputPath, nativeOutputPath);
+ assertTrue("job should complete successfully", nativeJob.runJob());
+
+ final String normalOutputPath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR
+ + "/" + keyclass.getName() + "/" + valueclass.getName();
+ // if output file exists ,then delete it
+ fs.delete(new Path(normalOutputPath), true);
+ hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
+ final KVJob normalJob = new KVJob(jobName, hadoopkvtestconf, keyclass,
+ valueclass, inputPath, normalOutputPath);
+ assertTrue("job should complete successfully", normalJob.runJob());
+
+ final boolean compareRet = ResultVerifier.verify(normalOutputPath,
+ nativeOutputPath);
+ assertEquals("job output not the same", true, compareRet);
+ ResultVerifier.verifyCounters(normalJob.job, nativeJob.job);
+ fs.close();
}
@AfterClass
@@ -137,35 +150,4 @@ public class KVTest {
fs.delete(new Path(TestConstants.NATIVETASK_KVTEST_DIR), true);
fs.close();
}
-
- private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
- final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(nativekvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- return outputpath;
- }
-
- private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
- final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- return outputpath;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
index 0202549..5f1619e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -78,11 +78,13 @@ public class LargeKVTest {
if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
return;
}
- final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
- deafult_KVSize_Maximum);
+ final int deafultKVSizeMaximum = 1 << 22; // 4M
+ final int kvSizeMaximum = normalConf.getInt(
+ TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
+ deafultKVSizeMaximum);
+ final FileSystem fs = FileSystem.get(normalConf);
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
+ for (int i = 65536; i <= kvSizeMaximum; i *= 4) {
int min = i / 4;
int max = i;
nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
@@ -90,48 +92,40 @@ public class LargeKVTest {
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- LOG.info("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
- + keyClass.getName() + ", valueClass: " + valueClass.getName());
-
- final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
- valueClass, nativeConf);
- final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
- valueClass, normalConf);
- final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
- final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
- + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
- + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
- assertEquals(reason, true, compareRet);
- }
- }
+ LOG.info("===KV Size Test: min size: " + min + ", max size: " + max
+ + ", keyClass: " + keyClass.getName() + ", valueClass: "
+ + valueClass.getName());
- private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws Exception {
- final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
- fs.close();
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- return outputpath;
- }
+ final String inputPath = TestConstants.NATIVETASK_KVTEST_INPUTDIR
+ + "/LargeKV/" + keyClass.getName() + "/" + valueClass.getName();
- private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws Exception {
- final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
+ final String nativeOutputPath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR
+ + "/LargeKV/" + keyClass.getName() + "/" + valueClass.getName();
+ // if output file exists ,then delete it
+ fs.delete(new Path(nativeOutputPath), true);
+ final KVJob nativeJob = new KVJob("Test Large Value Size:"
+ + String.valueOf(i), nativeConf, keyClass, valueClass, inputPath,
+ nativeOutputPath);
+ assertTrue("job should complete successfully", nativeJob.runJob());
+
+ final String normalOutputPath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR
+ + "/LargeKV/" + keyClass.getName() + "/" + valueClass.getName();
+ // if output file exists ,then delete it
+ fs.delete(new Path(normalOutputPath), true);
+ final KVJob normalJob = new KVJob("Test Large Key Size:" + String.valueOf(i),
+ normalConf, keyClass, valueClass, inputPath, normalOutputPath);
+ assertTrue("job should complete successfully", normalJob.runJob());
+
+ final boolean compareRet = ResultVerifier.verify(normalOutputPath,
+ nativeOutputPath);
+ final String reason = "keytype: " + keyClass.getName() + ", valuetype: "
+ + valueClass.getName() + ", failed with "
+ + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
+ + ", max size: " + max + ", normal out: " + normalOutputPath
+ + ", native Out: " + nativeOutputPath;
+ assertEquals(reason, true, compareRet);
+ ResultVerifier.verifyCounters(normalJob.job, nativeJob.job);
+ }
fs.close();
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- return outputpath;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index a8d2d92..2258726 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.nativetask.nonsorttest;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -53,19 +54,20 @@ public class NonSortTest {
final Job nativeNonSort = getJob(nativeConf, "NativeNonSort",
TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT);
- nativeNonSort.waitForCompletion(true);
+ assertTrue(nativeNonSort.waitForCompletion(true));
Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
final Job hadoopWithSort = getJob(normalConf, "NormalJob",
TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
- hadoopWithSort.waitForCompletion(true);
+ assertTrue(hadoopWithSort.waitForCompletion(true));
final boolean compareRet = ResultVerifier.verify(
TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT,
TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
+ ResultVerifier.verifyCounters(hadoopWithSort, nativeNonSort);
}
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
index b665971..8a7916b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapred.nativetask.testutil;
+import static org.junit.Assert.assertEquals;
+
import java.io.IOException;
import java.util.zip.CRC32;
@@ -25,6 +27,9 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.TaskCounter;
public class ResultVerifier {
/**
@@ -136,6 +141,23 @@ public class ResultVerifier {
return true;
}
- public static void main(String[] args) {
+ public static void verifyCounters(Job normalJob, Job nativeJob, boolean hasCombiner) throws IOException {
+ Counters normalCounters = normalJob.getCounters();
+ Counters nativeCounters = nativeJob.getCounters();
+ assertEquals("Counter MAP_OUTPUT_RECORDS should be equal",
+ normalCounters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue(),
+ nativeCounters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue());
+ assertEquals("Counter REDUCE_INPUT_GROUPS should be equal",
+ normalCounters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getValue(),
+ nativeCounters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getValue());
+ if (!hasCombiner) {
+ assertEquals("Counter REDUCE_INPUT_RECORDS should be equal",
+ normalCounters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue(),
+ nativeCounters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue());
+ }
+ }
+
+ public static void verifyCounters(Job normalJob, Job nativeJob) throws IOException {
+ verifyCounters(normalJob, nativeJob, false);
}
}
[45/50] [abbrv] git commit: YARN-2525. yarn logs command gives error
on trunk (Akira AJISAKA via aw)
Posted by ar...@apache.org.
YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40364dc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40364dc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40364dc4
Branch: refs/heads/HDFS-6581
Commit: 40364dc47c03efa295ae03fe8aa8467017fb6f26
Parents: 957414d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 12 10:35:41 2014 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 12 14:56:41 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40364dc4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3993f5e..9002e6a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -23,6 +23,8 @@ Trunk - Unreleased
YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
+ YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40364dc4/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 371d23d..12f7bb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -99,7 +99,7 @@ case "${COMMAND}" in
CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
;;
logs)
- CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+ CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
;;
[46/50] [abbrv] git commit: YARN-2456. Possible livelock in
CapacityScheduler when RM is recovering apps. Contributed by Jian He
Posted by ar...@apache.org.
YARN-2456. Possible livelock in CapacityScheduler when RM is recovering
apps. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e65ae575
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e65ae575
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e65ae575
Branch: refs/heads/HDFS-6581
Commit: e65ae575a059a426c4c38fdabe22a31eabbb349e
Parents: 40364dc
Author: XuanGong <xg...@apache.org>
Authored: Fri Sep 12 15:21:46 2014 -0700
Committer: XuanGong <xg...@apache.org>
Committed: Fri Sep 12 15:21:46 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../resourcemanager/recovery/RMStateStore.java | 3 +-
.../server/resourcemanager/TestRMRestart.java | 43 ++++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9002e6a..efc3e09 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -345,6 +345,9 @@ Release 2.6.0 - UNRELEASED
YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
FSData(In|Out)putStream in final block (Tsuyoshi OZAWA via jlowe)
+ YARN-2456. Possible livelock in CapacityScheduler when RM is recovering apps.
+ (Jian He via xgong)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index ac51a17..df4f3a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import javax.crypto.SecretKey;
@@ -421,7 +422,7 @@ public abstract class RMStateStore extends AbstractService {
*/
public static class RMState {
Map<ApplicationId, ApplicationState> appState =
- new HashMap<ApplicationId, ApplicationState>();
+ new TreeMap<ApplicationId, ApplicationState>();
RMDTSecretManagerState rmSecretManagerState = new RMDTSecretManagerState();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 7d511db..caa5647 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
@@ -1656,6 +1658,47 @@ public class TestRMRestart {
rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
}
+ @Test (timeout = 20000)
+ public void testAppRecoveredInOrderOnRMRestart() throws Exception {
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ for (int i = 10; i > 0; i--) {
+ ApplicationState appState = mock(ApplicationState.class);
+ when(appState.getAppId()).thenReturn(ApplicationId.newInstance(1234, i));
+ memStore.getState().getApplicationState()
+ .put(appState.getAppId(), appState);
+ }
+
+ MockRM rm1 = new MockRM(conf, memStore) {
+ @Override
+ protected RMAppManager createRMAppManager() {
+ return new TestRMAppManager(this.rmContext, this.scheduler,
+ this.masterService, this.applicationACLsManager, conf);
+ }
+
+ class TestRMAppManager extends RMAppManager {
+ ApplicationId prevId = ApplicationId.newInstance(1234, 0);
+
+ public TestRMAppManager(RMContext context, YarnScheduler scheduler,
+ ApplicationMasterService masterService,
+ ApplicationACLsManager applicationACLsManager, Configuration conf) {
+ super(context, scheduler, masterService, applicationACLsManager, conf);
+ }
+
+ @Override
+ protected void recoverApplication(ApplicationState appState,
+ RMState rmState) throws Exception {
+ // check application is recovered in order.
+ Assert.assertTrue(rmState.getApplicationState().size() > 0);
+ Assert.assertTrue(appState.getAppId().compareTo(prevId) > 0);
+ prevId = appState.getAppId();
+ }
+ }
+ };
+ rm1.start();
+ }
+
@SuppressWarnings("resource")
@Test (timeout = 60000)
public void testQueueMetricsOnRMRestart() throws Exception {
[30/50] [abbrv] git commit: MAPREDUCE-6056. native-task: move system
test working dir to target dir and cleanup test config xml files (Manu Zhang
via bchang)
Posted by ar...@apache.org.
MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ade9b04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ade9b04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ade9b04
Branch: refs/heads/HDFS-6581
Commit: 7ade9b04ea0a46a51564d23c8d715ddb1b9ca6d0
Parents: 17cd0fa
Author: Binglin Chang <bc...@apache.org>
Authored: Tue Sep 2 16:26:09 2014 +0800
Committer: Binglin Chang <bc...@apache.org>
Committed: Tue Sep 2 16:26:09 2014 +0800
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../nativetask/combinertest/CombinerTest.java | 65 ++++----
.../combinertest/LargeKVCombinerTest.java | 94 +++++------
.../combinertest/OldAPICombinerTest.java | 27 ++--
.../nativetask/compresstest/CompressMapper.java | 38 ++---
.../nativetask/compresstest/CompressTest.java | 90 +++++++----
.../TestNativeCollectorOnlyHandler.java | 20 ++-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 83 +++++-----
.../mapred/nativetask/kvtest/LargeKVTest.java | 104 ++++++------
.../nativetask/nonsorttest/NonSortTest.java | 30 ++--
.../nativetask/testutil/TestConstants.java | 56 ++++---
.../src/test/resources/common_conf.xml | 71 +++++----
.../src/test/resources/kvtest-conf.xml | 158 +++++++++----------
.../src/test/resources/native_conf.xml | 33 ++--
.../src/test/resources/normal_conf.xml | 26 ++-
.../src/test/resources/test-combiner-conf.xml | 54 +++----
.../src/test/resources/test-compress-conf.xml | 50 ++++++
.../test/resources/test-gzip-compress-conf.xml | 39 -----
.../test/resources/test-lz4-compress-conf.xml | 39 -----
.../src/test/resources/test-nonsort-conf.xml | 49 +++---
.../resources/test-snappy-compress-conf.xml | 41 -----
21 files changed, 588 insertions(+), 580 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index baa88c1..7c9558e 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -18,3 +18,4 @@ MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang v
MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
MAPREDUCE-6054. native-task: Speed up tests (todd)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
+MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index abbe28e..d7f05be 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -38,11 +38,14 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class CombinerTest {
private FileSystem fs;
private String inputpath;
@@ -50,33 +53,25 @@ public class CombinerTest {
private String hadoopoutputpath;
@Test
- public void testWordCountCombiner() {
- try {
-
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
-
- final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
- commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
- final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
-
- assertTrue(nativejob.waitForCompletion(true));
-
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertTrue(normaljob.waitForCompletion(true));
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
- assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
- nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
- }
+ public void testWordCountCombiner() throws Exception {
+ final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
+
+ final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
+ commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
+ final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
+
+ assertTrue(nativejob.waitForCompletion(true));
+
+ Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ assertTrue(normaljob.waitForCompletion(true));
+ Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
+
+ assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
+ assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
+ nativeReduceGroups.getValue(), normalReduceGroups.getValue());
}
@Before
@@ -88,8 +83,7 @@ public class CombinerTest {
this.fs = FileSystem.get(conf);
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
+ this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount";
if (!fs.exists(new Path(inputpath))) {
new TestInputFile(
@@ -98,10 +92,15 @@ public class CombinerTest {
Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
}
- this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
- this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
+ this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR + "/nativewordcount";
+ this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR + "/normalwordcount";
+ }
+
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+ fs.close();
}
protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index ccf0272..4ba4550 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -35,11 +35,14 @@ import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class LargeKVCombinerTest {
private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class);
@@ -50,61 +53,62 @@ public class LargeKVCombinerTest {
}
@Test
- public void testLargeValueCombiner(){
+ public void testLargeValueCombiner() throws Exception {
final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
+ final int KVSize_Maximum = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
deafult_KVSize_Maximum);
- final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
- final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
- final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
- try {
- final FileSystem fs = FileSystem.get(normalConf);
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
-
- int max = i;
- int min = Math.max(i / 4, max - 10);
-
- LOG.info("===KV Size Test: min size: " + min + ", max size: " + max);
-
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- fs.delete(new Path(inputPath), true);
- new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
- 1000000), IntWritable.class.getName(),
- Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
-
- final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
- final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
-
- assertTrue(nativejob.waitForCompletion(true));
+ final String inputPath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/largeKV";
+ final String nativeOutputPath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR
+ + "/nativeLargeKV";
+ final String hadoopOutputPath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR
+ + "/normalLargeKV";
+ final FileSystem fs = FileSystem.get(normalConf);
+ for (int i = 65536; i <= KVSize_Maximum; i *= 4) {
+
+ int max = i;
+ int min = Math.max(i / 4, max - 10);
+
+ LOG.info("===KV Size Test: min size: " + min + ", max size: " + max);
+
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ fs.delete(new Path(inputPath), true);
+ new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
+ 1000000), IntWritable.class.getName(),
+ Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
+
+ final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
+ final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
+
+ assertTrue(nativejob.waitForCompletion(true));
Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertTrue(normaljob.waitForCompletion(true));
+
+ assertTrue(normaljob.waitForCompletion(true));
Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-
- final String reason = "LargeKVCombinerTest failed with, min size: " + min
- + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
-
- assertEquals(reason, true, compareRet);
-// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
+
+ final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
+
+ final String reason = "LargeKVCombinerTest failed with, min size: " + min
+ + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
+
+ assertEquals(reason, true, compareRet);
+// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
// nativeReduceGroups.getValue(), normalReduceGroups.getValue());
- }
- fs.close();
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
}
+ fs.close();
+ }
+
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+ fs.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
index 19493c7..1c76bc6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
@@ -38,11 +38,14 @@ import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Counter;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class OldAPICombinerTest {
private FileSystem fs;
private String inputpath;
@@ -51,23 +54,23 @@ public class OldAPICombinerTest {
public void testWordCountCombinerWithOldAPI() throws Exception {
final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
+ final String nativeoutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH;
final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
RunningJob nativeRunning = JobClient.runJob(nativeJob);
Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
+
final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
+ final String normaloutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH;
final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
-
+
RunningJob normalRunning = JobClient.runJob(normalJob);
Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
+
final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-
+
assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
}
@@ -78,15 +81,21 @@ public class OldAPICombinerTest {
final ScenarioConfiguration conf = new ScenarioConfiguration();
conf.addcombinerConf();
this.fs = FileSystem.get(conf);
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
+ this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount";
if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
+ new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000), Text.class.getName(),
Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
}
}
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+ fs.close();
+ }
+
private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
throws Exception {
final JobConf jobConf = new JobConf(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
index 83c4794..1c91ff0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -32,8 +31,6 @@ import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class CompressMapper {
- public static final String inputFile = "./compress/input.txt";
- public static final String outputFileDir = "./compress/output/";
public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
@@ -43,27 +40,22 @@ public class CompressMapper {
}
}
- public static Job getCompressJob(String jobname, Configuration conf) {
- Job job = null;
- try {
- job = new Job(conf, jobname + "-CompressMapperJob");
- job.setJarByClass(CompressMapper.class);
- job.setMapperClass(TextCompressMapper.class);
- job.setOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Text.class);
- final Path outputpath = new Path(outputFileDir + jobname);
- // if output file exists ,delete it
- final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
- if (hdfs.exists(outputpath)) {
- hdfs.delete(outputpath);
- }
- hdfs.close();
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputFile));
- FileOutputFormat.setOutputPath(job, outputpath);
- } catch (final Exception e) {
- e.printStackTrace();
+ public static Job getCompressJob(String jobname, Configuration conf, String inputpath, String outputpath)
+ throws Exception {
+ Job job = new Job(conf, jobname + "-CompressMapperJob");
+ job.setJarByClass(CompressMapper.class);
+ job.setMapperClass(TextCompressMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Text.class);
+ // if output file exists ,delete it
+ final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
+ if (hdfs.exists(new Path(outputpath))) {
+ hdfs.delete(new Path(outputpath));
}
+ hdfs.close();
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ FileInputFormat.addInputPath(job, new Path(inputpath));
+ FileOutputFormat.setOutputPath(job, new Path(outputpath));
return job;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index 67321d6..3a03d29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -30,60 +30,87 @@ import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class CompressTest {
+ private static final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ private static final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
+
+ static {
+ nativeConf.addResource(TestConstants.COMPRESS_TEST_CONF_PATH);
+ hadoopConf.addResource(TestConstants.COMPRESS_TEST_CONF_PATH);
+ }
+
@Test
public void testSnappyCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
+ final String snappyCodec = "org.apache.hadoop.io.compress.SnappyCodec";
+
+ nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, snappyCodec);
+ final String nativeOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/snappy";
+ final Job job = CompressMapper.getCompressJob("nativesnappy", nativeConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
assertTrue(job.waitForCompletion(true));
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
+ hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, snappyCodec);
+ final String hadoopOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/snappy";
+ final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
assertTrue(hadoopjob.waitForCompletion(true));
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
- CompressMapper.outputFileDir + "hadoopsnappy");
+ final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
@Test
public void testGzipCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativegzip", conf);
+ final String gzipCodec = "org.apache.hadoop.io.compress.GzipCodec";
+
+ nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, gzipCodec);
+ final String nativeOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/gzip";
+ final Job job = CompressMapper.getCompressJob("nativegzip", nativeConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
assertTrue(job.waitForCompletion(true));
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
+ hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, gzipCodec);
+ final String hadoopOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/gzip";
+ final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
assertTrue(hadoopjob.waitForCompletion(true));
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
- CompressMapper.outputFileDir + "hadoopgzip");
+ final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
@Test
public void testLz4Compress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
+ final String lz4Codec = "org.apache.hadoop.io.compress.Lz4Codec";
+
+ nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, lz4Codec);
+ final String nativeOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/lz4";
+ final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
assertTrue(nativeJob.waitForCompletion(true));
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
+ hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, lz4Codec);
+ final String hadoopOutputPath =
+ TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/lz4";
+ final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf,
+ TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
assertTrue(hadoopJob.waitForCompletion(true));
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
- CompressMapper.outputFileDir + "hadooplz4");
+ final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
@@ -93,15 +120,22 @@ public class CompressTest {
Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
final ScenarioConfiguration conf = new ScenarioConfiguration();
final FileSystem fs = FileSystem.get(conf);
- final Path path = new Path(CompressMapper.inputFile);
+ final Path path = new Path(TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR);
fs.delete(path);
if (!fs.exists(path)) {
- new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
+ new TestInputFile(hadoopConf.getInt(
TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
Text.class.getName(), Text.class.getName(), conf)
- .createSequenceTestFile(CompressMapper.inputFile);
-
+ .createSequenceTestFile(TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR);
}
fs.close();
}
+
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_COMPRESS_TEST_DIR), true);
+ fs.close();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
index 05d87cf..6b9edc8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java
@@ -22,6 +22,9 @@ import java.io.IOException;
import junit.framework.Assert;
import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.Command;
@@ -30,6 +33,7 @@ import org.apache.hadoop.mapred.nativetask.INativeHandler;
import org.apache.hadoop.mapred.nativetask.TaskContext;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
import org.apache.hadoop.mapred.nativetask.util.OutputUtil;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.mockito.Matchers;
@@ -43,7 +47,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
private BufferPusher pusher;
private ICombineHandler combiner;
private TaskContext taskContext;
- private String localDir = "build/test/mapred/local";
+ private static final String LOCAL_DIR = TestConstants.NATIVETASK_TEST_DIR + "/local";
@Override
public void setUp() throws IOException {
@@ -53,7 +57,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
JobConf jobConf = new JobConf();
jobConf.set(OutputUtil.NATIVE_TASK_OUTPUT_MANAGER,
"org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles");
- jobConf.set("mapred.local.dir", localDir);
+ jobConf.set("mapred.local.dir", LOCAL_DIR);
this.taskContext = new TaskContext(jobConf,
BytesWritable.class, BytesWritable.class,
BytesWritable.class,
@@ -64,6 +68,12 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100));
}
+ @Override
+ public void tearDown() throws IOException {
+ FileSystem.getLocal(new Configuration()).delete(new Path(LOCAL_DIR));
+ }
+
+
public void testCollect() throws IOException {
this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
handler.collect(new BytesWritable(), new BytesWritable(), 100);
@@ -95,9 +105,9 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
}
Assert.assertTrue("exception thrown", thrown);
- final String expectedOutputPath = localDir + "/output/file.out";
- final String expectedOutputIndexPath = localDir + "/output/file.out.index";
- final String expectedSpillPath = localDir + "/output/spill0.out";
+ final String expectedOutputPath = LOCAL_DIR + "/output/file.out";
+ final String expectedOutputIndexPath = LOCAL_DIR + "/output/file.out.index";
+ final String expectedSpillPath = LOCAL_DIR + "/output/spill0.out";
final String outputPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString();
Assert.assertEquals(expectedOutputPath, outputPath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 1657c04..784c14f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
@@ -112,70 +113,58 @@ public class KVTest {
}
@Test
- public void testKVCompability() {
- try {
- final String nativeoutput = this.runNativeTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final String normaloutput = this.runNormalTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
- final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- if(compareRet){
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(nativeoutput), true);
- fs.delete(new Path(normaloutput), true);
- fs.delete(new Path(input), true);
- fs.close();
- }
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- } catch (final IOException e) {
- assertEquals("test run exception:", null, e);
- } catch (final Exception e) {
- assertEquals("test run exception:", null, e);
+ public void testKVCompability() throws Exception {
+ final String nativeoutput = this.runNativeTest(
+ "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
+ final String normaloutput = this.runNormalTest(
+ "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
+ final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
+ final String input = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ if(compareRet){
+ final FileSystem fs = FileSystem.get(hadoopkvtestconf);
+ fs.delete(new Path(nativeoutput), true);
+ fs.delete(new Path(normaloutput), true);
+ fs.delete(new Path(input), true);
+ fs.close();
}
+ assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
- private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
- + keyclass.getName() + "/" + valueclass.getName();
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_KVTEST_DIR), true);
+ fs.close();
+ }
+
+ private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
+ final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
// if output file exists ,then delete it
final FileSystem fs = FileSystem.get(nativekvtestconf);
fs.delete(new Path(outputpath));
fs.close();
nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- try {
- final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- } catch (final Exception e) {
- return "native testcase run time error.";
- }
+ final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
+ assertTrue("job should complete successfully", keyJob.runJob());
return outputpath;
}
- private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = hadoopkvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
- + "/"
+ private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
+ final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/"
+ keyclass.getName() + "/" + valueclass.getName();
// if output file exists ,then delete it
final FileSystem fs = FileSystem.get(hadoopkvtestconf);
fs.delete(new Path(outputpath));
fs.close();
hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- try {
- final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
+ final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
+ assertTrue("job should complete successfully", keyJob.runJob());
return outputpath;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
index 6b99907..0202549 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.nativetask.NativeRuntime;
import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
@@ -47,97 +48,90 @@ public class LargeKVTest {
Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
}
+ private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+ private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
+ static {
+ nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
+ nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
+ normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
+ normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
+ }
+
@Test
- public void testKeySize() {
+ public void testKeySize() throws Exception {
runKVSizeTests(Text.class, IntWritable.class);
}
@Test
- public void testValueSize() {
+ public void testValueSize() throws Exception {
runKVSizeTests(IntWritable.class, Text.class);
}
- private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- static {
- nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
- nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
- normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_KVTEST_DIR), true);
+ fs.close();
}
- public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
+ public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) throws Exception {
if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
return;
}
final int deafult_KVSize_Maximum = 1 << 22; // 4M
final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
deafult_KVSize_Maximum);
- try {
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
- int min = i / 4;
- int max = i;
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
+ int min = i / 4;
+ int max = i;
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
+ normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- LOG.info("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
- + keyClass.getName() + ", valueClass: " + valueClass.getName());
+ LOG.info("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
+ + keyClass.getName() + ", valueClass: " + valueClass.getName());
- final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
- valueClass, nativeConf);
- final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
- valueClass, normalConf);
- final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
- final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
- + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
- + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
- assertEquals(reason, true, compareRet);
- }
- } catch (final Exception e) {
- // TODO: handle exception
- // assertEquals("test run exception:", null, e);
- e.printStackTrace();
+ final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
+ valueClass, nativeConf);
+ final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
+ valueClass, normalConf);
+ final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
+ final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
+ + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
+ + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
+ assertEquals(reason, true, compareRet);
}
}
private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
throws Exception {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
+ final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/LargeKV/"
+ + keyclass.getName() + "/" + valueclass.getName();
// if output file exists ,then delete it
final FileSystem fs = FileSystem.get(conf);
fs.delete(new Path(outputpath), true);
fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
+ final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
+ assertTrue("job should complete successfully", keyJob.runJob());
return outputpath;
}
private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws IOException {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
+ throws Exception {
+ final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
+ + keyclass.getName() + "/" + valueclass.getName();
+ final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/LargeKV/"
+ + keyclass.getName() + "/" + valueclass.getName();
// if output file exists ,then delete it
final FileSystem fs = FileSystem.get(conf);
fs.delete(new Path(outputpath), true);
fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- assertTrue("job should complete successfully", keyJob.runJob());
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
+ final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
+ assertTrue("job should complete successfully", keyJob.runJob());
return outputpath;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index b1bb5b1..a8d2d92 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
@@ -49,20 +50,21 @@ public class NonSortTest {
Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
- String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
- final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
+ final Job nativeNonSort = getJob(nativeConf, "NativeNonSort",
+ TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
+ TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT);
nativeNonSort.waitForCompletion(true);
Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
- inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
- final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
+ final Job hadoopWithSort = getJob(normalConf, "NormalJob",
+ TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
+ TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
hadoopWithSort.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
- normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
+ final boolean compareRet = ResultVerifier.verify(
+ TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT,
+ TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
}
@@ -73,14 +75,22 @@ public class NonSortTest {
final ScenarioConfiguration configuration = new ScenarioConfiguration();
configuration.addNonSortTestConf();
final FileSystem fs = FileSystem.get(configuration);
- final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
+ final Path path = new Path(TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR);
if (!fs.exists(path)) {
- new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
+ new TestInputFile(configuration.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE, 10000000), Text.class.getName(),
Text.class.getName(), configuration).createSequenceTestFile(path.toString());
}
fs.close();
}
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+ fs.delete(new Path(TestConstants.NATIVETASK_NONSORT_TEST_DIR), true);
+ fs.close();
+ }
+
+
private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
final FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputpath))) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
index 3beb082..0b5a117 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
@@ -19,49 +19,53 @@ package org.apache.hadoop.mapred.nativetask.testutil;
public class TestConstants {
// conf path
+ public static final String COMMON_CONF_PATH = "common_conf.xml";
public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
+ public static final String COMPRESS_TEST_CONF_PATH = "test-compress-conf.xml";
+ // common constants
+ public static final String NATIVETASK_TEST_DIR = System.getProperty("test.build.data", "target/test/data");
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
+ public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
- public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
- public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
- public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
+ // kv test
+ public static final String FILESIZE_KEY = "kvtest.file.size";
+ public static final String NATIVETASK_KVTEST_DIR = NATIVETASK_TEST_DIR + "/kvtest";
+ public static final String NATIVETASK_KVTEST_INPUTDIR = NATIVETASK_KVTEST_DIR + "/input";
+ public static final String NATIVETASK_KVTEST_NATIVE_OUTPUTDIR = NATIVETASK_KVTEST_DIR + "/output/native";
+ public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = NATIVETASK_KVTEST_DIR + "/output/normal";
public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
- public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
- public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
- public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
- public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
- public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
- public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
- public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
+ // compress test
public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
+ public static final String NATIVETASK_COMPRESS_TEST_DIR = NATIVETASK_TEST_DIR + "/compresstest";
+ public static final String NATIVETASK_COMPRESS_TEST_INPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/input";
+ public static final String NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/output/native";
+ public static final String NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/output/normal";
- public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
- public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
- public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
- public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
- public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
- public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
- public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
- public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
+ // combiner test
+ public static final String NATIVETASK_COMBINER_TEST_DIR = NATIVETASK_TEST_DIR + "/combinertest";
+ public static final String NATIVETASK_COMBINER_TEST_INPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/input";
+ public static final String NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/output/native";
+ public static final String NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/output/normal";
+ public static final String NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH = NATIVETASK_COMBINER_TEST_DIR + "/oldapi/output/native";
+ public static final String NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH = NATIVETASK_COMBINER_TEST_DIR + "/oldapi/output/normal";
public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
- public static final String COMMON_CONF_PATH = "common_conf.xml";
-
- public static final String FILESIZE_KEY = "kvtest.file.size";
- public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
-
+ // nonsort test
public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
- public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
- public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
- public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
+ public static final String NATIVETASK_NONSORT_TEST_DIR = NATIVETASK_TEST_DIR + "/nonsorttest";
+ public static final String NATIVETASK_NONSORT_TEST_INPUTDIR = NATIVETASK_NONSORT_TEST_DIR + "/input";
+ public static final String NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT = NATIVETASK_NONSORT_TEST_DIR + "/output/native";
+ public static final String NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT = NATIVETASK_NONSORT_TEST_DIR + "/output/normal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml
index c09e17a..d4574d5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml
@@ -1,39 +1,48 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!-- Put site-specific property overrides in this file. -->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
-<property>
-<name>mapred.job.tracker</name>
-<value>local</value>
-</property>
-
-<property>
-<name>keep.failed.task.files</name>
-<value>true</value>
-</property>
-
<property>
- <name>keep.task.files.pattern</name>
- <value>.*_m_</value>
- <description>Keep all files from tasks whose task names match the given
- regular expression. Defaults to none.</description>
+ <name>mapred.job.tracker</name>
+ <value>local</value>
+ </property>
+ <property>
+ <name>keep.failed.task.files</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>keep.task.files.pattern</name>
+ <value>.*_m_</value>
+ <description>Keep all files from tasks whose task names match the given
+ regular expression. Defaults to none.</description>
+ </property>
+ <property>
+ <name>nativetask.kvsize.max.largekv</name>
+ <value>1048576</value>
+ </property>
+ <property>
+ <name>native.processor.buffer.kb</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>nativetask.output.manager</name>
+ <value>org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles</value>
</property>
-
-<property>
-<name>nativetask.kvsize.max.largekv</name>
-<value>1048576</value>
-</property>
-
-<property>
-<name>native.processor.buffer.kb</name>
-<value>128</value>
-</property>
-
-<property>
-<name>nativetask.output.manager</name>
-<value>org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles</value>
-</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml
index addcbc7..eb626d3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml
@@ -1,87 +1,85 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>nativetask.kvtest.inputdir</name>
-<value>./kvtest/input</value>
-</property>
-<property>
-<name>nativetask.kvtest.outputdir</name>
-<value>./kvtest/output/native</value>
-</property>
-<property>
-<name>normal.kvtest.outputdir</name>
-<value>./kvtest/output/normal</value>
-</property>
-<property>
-<name>kvtest.file.size</name>
-<value>10485760</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.kvtest.keyclasses</name>
-<value>
-org.apache.hadoop.io.BytesWritable;
-org.apache.hadoop.io.BooleanWritable;
-org.apache.hadoop.io.ByteWritable;
-org.apache.hadoop.io.DoubleWritable;
-org.apache.hadoop.io.FloatWritable;
-org.apache.hadoop.io.IntWritable;
-org.apache.hadoop.io.LongWritable;
-org.apache.hadoop.io.Text;
-org.apache.hadoop.io.VIntWritable;
-org.apache.hadoop.io.VLongWritable;
-</value>
-</property>
-<property>
-<name>nativetask.kvtest.valueclasses</name>
-<value>
-org.apache.hadoop.io.BytesWritable;
-org.apache.hadoop.io.BooleanWritable;
-org.apache.hadoop.io.ByteWritable;
-org.apache.hadoop.io.DoubleWritable;
-org.apache.hadoop.io.FloatWritable;
-org.apache.hadoop.io.IntWritable;
-org.apache.hadoop.io.LongWritable;
-org.apache.hadoop.io.Text;
-org.apache.hadoop.io.VIntWritable;
-org.apache.hadoop.io.VLongWritable;
-org.apache.hadoop.mapred.nativetask.testutil.MockValueClass;
-</value>
-</property>
-
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.SnappyCodec</value>
-</property>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
-
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
+<configuration>
+ <property>
+ <name>kvtest.file.size</name>
+ <value>10485760</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.class</name>
+ <value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
+ </property>
+ <property>
+ <name>mapred.output.value.class</name>
+ <value>org.apache.hadoop.io.IntWritable</value>
+ </property>
+ <property>
+ <name>nativetask.kvtest.keyclasses</name>
+ <value>
+ org.apache.hadoop.io.BytesWritable;
+ org.apache.hadoop.io.BooleanWritable;
+ org.apache.hadoop.io.ByteWritable;
+ org.apache.hadoop.io.DoubleWritable;
+ org.apache.hadoop.io.FloatWritable;
+ org.apache.hadoop.io.IntWritable;
+ org.apache.hadoop.io.LongWritable;
+ org.apache.hadoop.io.Text;
+ org.apache.hadoop.io.VIntWritable;
+ org.apache.hadoop.io.VLongWritable;
+ </value>
+ </property>
+ <property>
+ <name>nativetask.kvtest.valueclasses</name>
+ <value>
+ org.apache.hadoop.io.BytesWritable;
+ org.apache.hadoop.io.BooleanWritable;
+ org.apache.hadoop.io.ByteWritable;
+ org.apache.hadoop.io.DoubleWritable;
+ org.apache.hadoop.io.FloatWritable;
+ org.apache.hadoop.io.IntWritable;
+ org.apache.hadoop.io.LongWritable;
+ org.apache.hadoop.io.Text;
+ org.apache.hadoop.io.VIntWritable;
+ org.apache.hadoop.io.VLongWritable;
+ org.apache.hadoop.mapred.nativetask.testutil.MockValueClass;
+ </value>
+ </property>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ </property>
+ <property>
+ <name>hadoop.native.lib</name>
+ <value>true</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml
index 0634d05..7d0d01f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml
@@ -1,15 +1,30 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-<name>mapreduce.job.map.output.collector.class</name>
-<value>org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator</value>
-</property>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
-<property>
-<name>io.sort.mb</name>
-<value>5</value>
-</property>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>mapreduce.job.map.output.collector.class</name>
+ <value>org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator</value>
+ </property>
+ <property>
+ <name>io.sort.mb</name>
+ <value>5</value>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml
index 38cd7d4..48fddd9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml
@@ -1,10 +1,26 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-<name>io.sort.mb</name>
-<value>30</value>
-</property>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>io.sort.mb</name>
+ <value>30</value>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml
index 53370fa..8f9ee4d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml
@@ -1,36 +1,30 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!-- Put site-specific property overrides in this file. -->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
-<configuration>
-<property>
-<name>nativetask.combinertest.inputpath</name>
-<value>./combinertest/input</value>
-</property>
-<property>
-<name>nativetask.combiner.wordcount.filesize</name>
-<value>20971520</value>
-</property>
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
-<property>
-<name>nativetask.combinertest.outputdir</name>
-<value>./combinertest/output/native</value>
-</property>
-<property>
-<name>normal.combinertest.outputdir</name>
-<value>./combinertest/output/normal</value>
-</property>
-<property>
-<name>oldAPI.nativetask.combinertest.outputdir</name>
-<value>./combinertest/oldapi/output/native</value>
-</property>
-<property>
-<name>oldAPI.normal.combinertest.outputdir</name>
-<value>./combinertest/oldapi/output/normal</value>
-</property>
-<property>
-<name>mapred.job.tracker</name>
-<value>local</value>
-</property>
+<configuration>
+ <property>
+ <name>nativetask.combiner.wordcount.filesize</name>
+ <value>20971520</value>
+ </property>
+ <property>
+ <name>mapred.job.tracker</name>
+ <value>local</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-compress-conf.xml
new file mode 100644
index 0000000..88d6cd8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-compress-conf.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapred.output.value.class</name>
+ <value>org.apache.hadoop.io.IntWritable</value>
+ </property>
+ <property>
+ <name>nativetask.compress.filesize</name>
+ <value>100000</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.class</name>
+ <value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
+ </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ </property>
+ <property>
+ <name>hadoop.native.lib</name>
+ <value>true</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml
deleted file mode 100644
index ed43034..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml
deleted file mode 100644
index 6e54c0b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.Lz4Codec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml
index f1308c3..f1f4c6d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml
@@ -1,31 +1,30 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!-- Put site-specific property overrides in this file. -->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
-<property>
-<name>nativetask.nonsorttest.inputpath</name>
-<value>./sorttest/input</value>
-</property>
-<property>
-<name>nativetask.nonsorttest.filesize</name>
-<value>4194304</value>
-</property>
-<property>
-<name>nonsorttest.native.outputdir.tmp</name>
-<value>./sorttest/output/tmp</value>
-</property>
-<property>
-<name>nonsorttest.native.outputdir</name>
-<value>./sorttest/output/native</value>
-</property>
-<property>
-<name>nonsorttest.normal.outputdir</name>
-<value>./sort/output/normal</value>
-</property>
-<property>
-<name>mapred.job.tracker</name>
-<value>local</value>
-</property>
+ <property>
+ <name>nativetask.nonsorttest.filesize</name>
+ <value>4194304</value>
+ </property>
+ <property>
+ <name>mapred.job.tracker</name>
+ <value>local</value>
+ </property>
</configuration>
[13/50] [abbrv] git commit: Add entry to branch's CHANGES.txt for
previous commit
Posted by ar...@apache.org.
Add entry to branch's CHANGES.txt for previous commit
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613827 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43917e56
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43917e56
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43917e56
Branch: refs/heads/HDFS-6581
Commit: 43917e564d266b67c323f4657d11a8b70ae1d706
Parents: ea9e5b7
Author: Todd Lipcon <to...@apache.org>
Authored: Sun Jul 27 18:54:02 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Sun Jul 27 18:54:02 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43917e56/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index aa695cf..92c94a8 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -6,3 +6,4 @@ MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd)
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
+MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
[02/50] [abbrv] git commit: Merge trunk into branch
Posted by ar...@apache.org.
Merge trunk into branch
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1612742 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/341695e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/341695e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/341695e7
Branch: refs/heads/HDFS-6581
Commit: 341695e7317d156c8cd869f52978100d4b88a6ea
Parents: e788e8b e0f00e1
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Jul 23 01:47:28 2014 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Jul 23 01:47:28 2014 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-auth/pom.xml | 11 +
hadoop-common-project/hadoop-common/CHANGES.txt | 81 +-
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 1 +
.../hadoop/crypto/key/CachingKeyProvider.java | 174 +++++
.../crypto/key/KeyProviderCryptoExtension.java | 235 ++++--
.../hadoop/crypto/key/KeyProviderFactory.java | 10 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 41 +-
.../crypto/key/kms/KMSClientProvider.java | 161 +++-
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 8 +
.../hadoop/crypto/key/kms/ValueQueue.java | 317 ++++++++
.../fs/CommonConfigurationKeysPublic.java | 33 +
.../main/java/org/apache/hadoop/fs/Stat.java | 13 +-
.../hadoop/fs/s3native/NativeS3FileSystem.java | 9 +-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 34 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 42 +
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 89 +++
.../main/java/org/apache/hadoop/io/Text.java | 19 +-
.../apache/hadoop/io/compress/BZip2Codec.java | 6 +-
.../hadoop/io/compress/CompressionCodec.java | 55 ++
.../io/compress/CompressionInputStream.java | 10 +
.../io/compress/CompressionOutputStream.java | 18 +-
.../apache/hadoop/io/compress/DefaultCodec.java | 14 +-
.../apache/hadoop/io/compress/GzipCodec.java | 14 +-
.../org/apache/hadoop/io/compress/Lz4Codec.java | 6 +-
.../apache/hadoop/io/compress/SnappyCodec.java | 6 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 29 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 12 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 41 +-
.../apache/hadoop/metrics2/MetricsSystem.java | 6 +
.../hadoop/metrics2/impl/MetricsConfig.java | 2 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 11 +
.../org/apache/hadoop/net/NetworkTopology.java | 9 +-
.../net/NetworkTopologyWithNodeGroup.java | 7 +-
.../java/org/apache/hadoop/security/Groups.java | 52 +-
.../hadoop/security/LdapGroupsMapping.java | 6 +-
.../security/ShellBasedUnixGroupsMapping.java | 3 +-
.../hadoop/security/alias/CredentialShell.java | 4 +-
.../security/alias/JavaKeyStoreProvider.java | 3 +
.../authorize/DefaultImpersonationProvider.java | 65 +-
.../authorize/ImpersonationProvider.java | 15 +
.../hadoop/security/authorize/ProxyUsers.java | 29 +-
.../main/java/org/apache/hadoop/util/Timer.java | 51 ++
.../main/java/org/apache/hadoop/util/Tool.java | 2 +-
.../src/main/resources/core-default.xml | 47 ++
.../src/site/apt/CommandsManual.apt.vm | 4 +-
.../src/site/apt/NativeLibraries.apt.vm | 2 +
.../crypto/key/TestCachingKeyProvider.java | 152 ++++
.../key/TestKeyProviderCryptoExtension.java | 124 ++-
.../apache/hadoop/crypto/key/TestKeyShell.java | 16 +-
.../hadoop/crypto/key/TestValueQueue.java | 190 +++++
.../java/org/apache/hadoop/fs/TestStat.java | 28 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 55 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 52 ++
.../java/org/apache/hadoop/io/TestText.java | 22 +
.../org/apache/hadoop/ipc/MiniRPCBenchmark.java | 8 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 2 +
.../metrics2/impl/TestGangliaMetrics.java | 12 +-
.../metrics2/impl/TestMetricsSystemImpl.java | 50 +-
.../net/TestNetworkTopologyWithNodeGroup.java | 8 +-
.../hadoop/security/TestDoAsEffectiveUser.java | 23 +-
.../hadoop/security/TestGroupsCaching.java | 56 ++
.../hadoop/security/alias/TestCredShell.java | 20 +-
.../security/authorize/TestProxyUsers.java | 126 ++-
.../java/org/apache/hadoop/util/FakeTimer.java | 52 ++
.../hadoop-kms/src/main/conf/kms-acls.xml | 15 +
...rKeyGeneratorKeyProviderCryptoExtension.java | 149 ++++
.../hadoop/crypto/key/kms/server/KMS.java | 97 ++-
.../hadoop/crypto/key/kms/server/KMSACLs.java | 36 +-
.../key/kms/server/KMSCacheKeyProvider.java | 177 -----
.../crypto/key/kms/server/KMSConfiguration.java | 14 +-
.../key/kms/server/KMSServerJSONUtils.java | 21 +-
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 50 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 110 ++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 111 ++-
.../key/kms/server/TestKMSCacheKeyProvider.java | 120 ---
.../java/org/apache/hadoop/nfs/NfsExports.java | 25 +-
.../org/apache/hadoop/nfs/TestNfsExports.java | 12 +
.../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 2 +
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 20 +-
.../nfs/nfs3/TestClientAccessPrivilege.java | 120 +++
.../hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java | 7 +
.../hadoop/hdfs/nfs/nfs3/TestReaddir.java | 8 +-
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 6 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 58 ++
.../bkjournal/TestBootstrapStandbyWithBKJM.java | 169 ++++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 50 +-
.../java/org/apache/hadoop/hdfs/HAUtil.java | 35 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 57 +-
.../hadoop/hdfs/protocol/DatanodeInfo.java | 2 +-
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../datatransfer/DataTransferProtocol.java | 21 +-
.../hdfs/protocol/datatransfer/Receiver.java | 11 +-
.../hdfs/protocol/datatransfer/Sender.java | 11 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 29 +-
.../NamenodeProtocolTranslatorPB.java | 8 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 67 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../server/blockmanagement/BlockManager.java | 38 +-
.../blockmanagement/BlockPlacementPolicy.java | 65 +-
.../BlockPlacementPolicyDefault.java | 59 +-
.../BlockPlacementPolicyWithNodeGroup.java | 40 +-
.../server/blockmanagement/DatanodeManager.java | 11 +-
.../blockmanagement/DatanodeStatistics.java | 6 +
.../blockmanagement/DatanodeStorageInfo.java | 18 +
.../blockmanagement/HeartbeatManager.java | 21 +-
.../hdfs/server/common/HdfsServerConstants.java | 3 +-
.../hdfs/server/datanode/BPOfferService.java | 3 +-
.../hdfs/server/datanode/BlockReceiver.java | 10 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 150 +++-
.../hdfs/server/datanode/DataXceiver.java | 25 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 9 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 74 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 21 +-
.../web/resources/DatanodeWebHdfsMethods.java | 3 +-
.../hdfs/server/namenode/FSClusterStats.java | 9 +
.../hadoop/hdfs/server/namenode/FSImage.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 35 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 19 +-
.../hdfs/server/namenode/NamenodeFsck.java | 9 +-
.../hdfs/server/namenode/XAttrStorage.java | 40 +-
.../server/namenode/ha/BootstrapStandby.java | 7 +-
.../web/resources/NamenodeWebHdfsMethods.java | 117 ++-
.../hdfs/server/protocol/BlockCommand.java | 15 +-
.../hdfs/tools/DFSZKFailoverController.java | 5 +
.../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 31 +-
.../web/resources/ExcludeDatanodesParam.java | 42 +
.../src/main/proto/DatanodeProtocol.proto | 1 +
.../src/main/proto/datatransfer.proto | 4 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 7 +
.../src/main/resources/hdfs-default.xml | 13 +
.../src/main/webapps/hdfs/explorer.html | 31 +-
.../site/apt/HDFSHighAvailabilityWithQJM.apt.vm | 4 +-
.../src/site/apt/HdfsMultihoming.apt.vm | 145 ++++
.../fs/viewfs/TestViewFileSystemWithAcls.java | 190 +++++
.../hadoop/fs/viewfs/TestViewFsWithAcls.java | 190 +++++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 7 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 54 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 17 +-
.../hadoop/hdfs/TestIsMethodSupported.java | 34 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 4 +-
.../TestDelegationTokenForProxyUser.java | 6 +-
.../blockmanagement/TestReplicationPolicy.java | 41 +-
.../TestReplicationPolicyConsiderLoad.java | 24 +-
.../TestReplicationPolicyWithNodeGroup.java | 37 +-
.../hdfs/server/common/TestJspHelper.java | 6 +-
.../server/datanode/BlockReportTestBase.java | 5 +-
.../server/datanode/SimulatedFSDataset.java | 10 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../server/datanode/TestBlockReplacement.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 5 +-
.../server/datanode/TestSimulatedFSDataset.java | 4 +-
.../fsdataset/impl/TestWriteToReplica.java | 27 +-
.../namenode/TestFavoredNodesEndToEnd.java | 62 +-
.../namenode/TestNamenodeCapacityReport.java | 186 ++++-
.../namenode/TestStartupOptionUpgrade.java | 19 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 13 +-
.../namenode/ha/TestDelegationTokensWithHA.java | 10 +-
.../web/resources/TestWebHdfsDataLocality.java | 89 ++-
.../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 42 +-
.../apache/hadoop/net/TestNetworkTopology.java | 45 +-
.../security/TestRefreshUserMappings.java | 6 +-
hadoop-mapreduce-project/CHANGES.txt | 18 +
.../bin/mr-jobhistory-daemon.sh | 1 +
.../hadoop/mapred/LocalContainerLauncher.java | 79 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 268 +++++--
.../v2/app/commit/CommitterEventHandler.java | 27 +-
.../v2/app/local/LocalContainerAllocator.java | 6 +
.../mapreduce/v2/app/rm/RMCommunicator.java | 34 +-
.../v2/app/rm/RMContainerAllocator.java | 12 +
.../v2/app/rm/RMContainerRequestor.java | 36 +-
.../mapred/TestLocalContainerLauncher.java | 73 ++
.../v2/app/rm/TestRMContainerAllocator.java | 244 ++++++
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 46 +-
.../lib/input/CombineFileInputFormat.java | 2 +-
.../lib/input/TestCombineFileInputFormat.java | 55 ++
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 88 ++-
.../apache/hadoop/mapred/ShuffleHandler.java | 84 +-
.../hadoop/mapred/TestShuffleHandler.java | 90 +++
hadoop-project/src/site/site.xml | 1 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 27 +-
.../metrics/AzureFileSystemMetricsSystem.java | 19 +-
.../fs/azure/AzureBlobStorageTestAccount.java | 4 +-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 7 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 1 +
.../org/apache/hadoop/tools/OptionsParser.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 44 ++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 36 +-
.../hadoop-yarn/bin/yarn-daemon.sh | 1 +
.../distributedshell/ApplicationMaster.java | 52 +-
.../applications/distributedshell/Client.java | 20 +-
.../yarn/security/AMRMTokenIdentifier.java | 14 +
.../org/apache/hadoop/yarn/util/FSDownload.java | 5 +-
.../hadoop/yarn/event/DrainDispatcher.java | 14 +-
.../apache/hadoop/yarn/util/TestFSDownload.java | 7 +
.../security/BaseNMTokenSecretManager.java | 2 +-
.../nodemanager/DefaultContainerExecutor.java | 20 +-
.../nodemanager/LinuxContainerExecutor.java | 19 +-
.../yarn/server/nodemanager/NodeManager.java | 13 +-
.../recovery/NMLeveldbStateStoreService.java | 195 ++++-
.../recovery/NMNullStateStoreService.java | 28 +
.../recovery/NMStateStoreService.java | 35 +
.../recovery/records/NMDBSchemaVersion.java | 80 ++
.../impl/pb/NMDBSchemaVersionPBImpl.java | 81 ++
.../security/NMTokenSecretManagerInNM.java | 116 ++-
.../server/nodemanager/webapp/NodePage.java | 2 +-
.../yarn_server_nodemanager_recovery.proto | 6 +
.../TestDefaultContainerExecutor.java | 121 ++-
.../TestLinuxContainerExecutorWithMocks.java | 41 +-
.../recovery/NMMemoryStateStoreService.java | 49 ++
.../TestNMLeveldbStateStoreService.java | 114 +++
.../security/TestNMTokenSecretManagerInNM.java | 154 ++++
.../hadoop-yarn-server-resourcemanager/pom.xml | 15 +
.../server/resourcemanager/ResourceManager.java | 4 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 26 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 32 +-
.../rmcontainer/RMContainer.java | 5 +
.../rmcontainer/RMContainerImpl.java | 28 +-
.../scheduler/AbstractYarnScheduler.java | 39 +
.../scheduler/AppSchedulingInfo.java | 69 +-
.../scheduler/SchedulerApplicationAttempt.java | 9 +-
.../scheduler/capacity/CapacityScheduler.java | 47 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 10 +-
.../scheduler/event/AppAddedSchedulerEvent.java | 10 +
.../event/AppAttemptAddedSchedulerEvent.java | 12 +-
.../scheduler/fair/FSSchedulerApp.java | 10 +-
.../scheduler/fair/FairScheduler.java | 114 ++-
.../fair/policies/FairSharePolicy.java | 6 +-
.../scheduler/fifo/FifoScheduler.java | 48 +-
.../security/AMRMTokenSecretManager.java | 232 ++++--
.../RMDelegationTokenSecretManager.java | 12 +
.../resourcemanager/webapp/RMWebServices.java | 279 ++++++-
.../webapp/dao/DelegationToken.java | 99 +++
.../resourcemanager/TestApplicationCleanup.java | 101 ++-
.../resourcemanager/TestFifoScheduler.java | 4 +-
.../server/resourcemanager/TestRMRestart.java | 9 +-
.../TestWorkPreservingRMRestart.java | 30 +
.../recovery/RMStateStoreTestBase.java | 14 +-
.../attempt/TestRMAppAttemptTransitions.java | 20 +-
.../rmcontainer/TestRMContainerImpl.java | 42 +
.../capacity/TestCapacityScheduler.java | 67 ++
.../scheduler/fair/FairSchedulerTestBase.java | 25 +-
.../scheduler/fair/TestFairScheduler.java | 146 +++-
.../security/TestAMRMTokens.java | 81 +-
.../TestRMWebServicesDelegationTokens.java | 775 +++++++++++++++++++
.../src/site/apt/ResourceManagerRest.apt.vm | 220 ++++++
.../src/site/apt/YarnCommands.apt.vm | 9 +-
249 files changed, 9962 insertions(+), 1859 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/341695e7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
----------------------------------------------------------------------