You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/09/15 19:49:44 UTC
[35/52] [abbrv] git commit: MAPREDUCE-6069. Lint/style fixes and
removal of unused code. Contributed by Todd Lipcon.
MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683987be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683987be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683987be
Branch: refs/heads/HDFS-6584
Commit: 683987be7c160e67ddb8534eeb3c464bbe2796dd
Parents: 0032216
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Sep 3 13:07:24 2014 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 5 10:44:38 2014 -0700
----------------------------------------------------------------------
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/LICENSE | 22 --
.../src/CMakeLists.txt | 6 -
.../hadoop/mapred/nativetask/Constants.java | 13 -
.../hadoop/mapred/nativetask/DataReceiver.java | 3 -
.../mapred/nativetask/ICombineHandler.java | 2 -
.../mapred/nativetask/INativeHandler.java | 8 -
.../mapred/nativetask/NativeBatchProcessor.java | 11 -
.../NativeMapOutputCollectorDelegator.java | 10 +-
.../hadoop/mapred/nativetask/NativeRuntime.java | 32 +-
.../hadoop/mapred/nativetask/Platform.java | 11 +-
.../nativetask/handlers/BufferPullee.java | 3 +-
.../nativetask/handlers/BufferPuller.java | 2 -
.../nativetask/handlers/BufferPushee.java | 6 +-
.../nativetask/handlers/BufferPusher.java | 3 +-
.../nativetask/handlers/CombinerHandler.java | 16 +-
.../mapred/nativetask/handlers/IDataLoader.java | 1 -
.../handlers/NativeCollectorOnlyHandler.java | 6 +-
.../serde/BytesWritableSerializer.java | 3 +-
.../mapred/nativetask/serde/IKVSerializer.java | 31 +-
.../mapred/nativetask/serde/KVSerializer.java | 6 +-
.../nativetask/serde/NativeSerialization.java | 11 +-
.../mapred/nativetask/util/BytesUtil.java | 2 +-
.../nativetask/util/LocalJobOutputFiles.java | 58 +---
.../nativetask/util/NativeTaskOutput.java | 55 +---
.../nativetask/util/NativeTaskOutputFiles.java | 72 ++---
.../mapred/nativetask/util/ReadWriteBuffer.java | 16 +-
.../src/main/native/cityhash/city.cc | 307 -------------------
.../src/main/native/cityhash/city.h | 90 ------
.../src/main/native/src/NativeTask.h | 137 ---------
.../src/main/native/src/codec/BlockCodec.cc | 5 +-
.../src/main/native/src/codec/BlockCodec.h | 2 +-
.../src/main/native/src/codec/GzipCodec.cc | 2 +-
.../src/main/native/src/codec/GzipCodec.h | 2 +-
.../src/main/native/src/codec/Lz4Codec.cc | 2 +-
.../src/main/native/src/codec/Lz4Codec.h | 2 +-
.../src/main/native/src/codec/SnappyCodec.cc | 2 +-
.../src/main/native/src/codec/SnappyCodec.h | 2 +-
.../native/src/handler/AbstractMapHandler.cc | 9 +-
.../src/main/native/src/handler/BatchHandler.cc | 9 +-
.../main/native/src/handler/CombineHandler.cc | 4 +-
.../main/native/src/handler/CombineHandler.h | 2 +-
.../src/handler/MCollectorOutputHandler.cc | 6 +-
.../src/main/native/src/lib/BufferStream.cc | 116 +------
.../src/main/native/src/lib/BufferStream.h | 39 +--
.../src/main/native/src/lib/Buffers.cc | 77 +----
.../src/main/native/src/lib/Buffers.h | 63 +---
.../src/main/native/src/lib/Combiner.cc | 73 -----
.../src/main/native/src/lib/Combiner.h | 18 +-
.../src/main/native/src/lib/Compressions.cc | 6 +-
.../src/main/native/src/lib/Compressions.h | 2 +-
.../src/main/native/src/lib/FileSystem.cc | 23 +-
.../src/main/native/src/lib/FileSystem.h | 2 +-
.../src/main/native/src/lib/IFile.cc | 6 +-
.../src/main/native/src/lib/IFile.h | 10 +-
.../src/main/native/src/lib/Iterator.cc | 6 +-
.../src/main/native/src/lib/Log.cc | 2 +-
.../main/native/src/lib/MapOutputCollector.cc | 58 ++--
.../main/native/src/lib/MapOutputCollector.h | 16 +-
.../src/main/native/src/lib/MapOutputSpec.cc | 4 +-
.../src/main/native/src/lib/MapOutputSpec.h | 4 +-
.../src/main/native/src/lib/MemoryBlock.cc | 24 +-
.../src/main/native/src/lib/MemoryPool.h | 4 +-
.../src/main/native/src/lib/Merge.cc | 5 +-
.../src/main/native/src/lib/Merge.h | 8 +-
.../src/main/native/src/lib/MinHeap.h | 2 +-
.../src/main/native/src/lib/NativeLibrary.cc | 7 +-
.../main/native/src/lib/NativeObjectFactory.cc | 19 +-
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 27 +-
.../src/main/native/src/lib/NativeTask.cc | 44 +--
.../src/main/native/src/lib/PartitionBucket.cc | 25 +-
.../src/main/native/src/lib/PartitionBucket.h | 16 +-
.../native/src/lib/PartitionBucketIterator.cc | 22 +-
.../native/src/lib/PartitionBucketIterator.h | 16 +-
.../src/main/native/src/lib/Path.cc | 2 +-
.../src/main/native/src/lib/SpillInfo.cc | 12 +-
.../src/main/native/src/lib/Streams.cc | 6 +-
.../src/main/native/src/lib/TaskCounters.cc | 3 +-
.../src/main/native/src/lib/TrackingCollector.h | 50 ---
.../src/main/native/src/lib/commons.h | 9 +-
.../src/main/native/src/lib/jniutils.cc | 8 +-
.../src/main/native/src/util/Checksum.cc | 2 +-
.../src/main/native/src/util/Hash.cc | 24 --
.../src/main/native/src/util/Hash.h | 73 -----
.../src/main/native/src/util/Random.cc | 4 +-
.../src/main/native/src/util/StringUtil.cc | 9 +-
.../src/main/native/src/util/SyncUtils.cc | 114 +------
.../src/main/native/src/util/SyncUtils.h | 228 --------------
.../src/main/native/src/util/Timer.cc | 7 +-
.../src/main/native/src/util/WritableUtils.cc | 6 +-
.../src/main/native/src/util/WritableUtils.h | 2 +-
.../src/main/native/test/TestCommand.cc | 9 +-
.../src/main/native/test/TestCompressions.cc | 14 +-
.../src/main/native/test/TestConfig.cc | 6 +-
.../src/main/native/test/TestCounter.cc | 8 +-
.../src/main/native/test/TestFileSystem.cc | 2 +-
.../src/main/native/test/TestIFile.cc | 15 +-
.../src/main/native/test/TestMain.cc | 21 +-
.../src/main/native/test/TestPrimitives.cc | 2 +-
.../src/main/native/test/TestSort.cc | 33 +-
.../src/main/native/test/lib/TestByteArray.cc | 2 +-
.../src/main/native/test/lib/TestByteBuffer.cc | 2 +-
.../lib/TestComparatorForDualPivotQuickSort.cc | 2 +-
.../native/test/lib/TestComparatorForStdSort.cc | 2 +-
.../native/test/lib/TestFixSizeContainer.cc | 2 +-
.../src/main/native/test/lib/TestIterator.cc | 4 +-
.../src/main/native/test/lib/TestKVBuffer.cc | 2 +-
.../native/test/lib/TestMemBlockIterator.cc | 11 +-
.../src/main/native/test/lib/TestMemoryBlock.cc | 8 +-
.../src/main/native/test/lib/TestMemoryPool.cc | 14 +-
.../main/native/test/lib/TestPartitionBucket.cc | 36 +--
.../src/main/native/test/lib/TestReadBuffer.cc | 6 +-
.../main/native/test/lib/TestReadWriteBuffer.cc | 2 +-
.../native/test/lib/TestTrackingCollector.cc | 37 ---
.../src/main/native/test/test_commons.cc | 4 +-
.../src/main/native/test/test_commons.h | 4 +-
.../src/main/native/test/util/TestHash.cc | 52 ----
.../src/main/native/test/util/TestSyncUtils.cc | 116 -------
.../nativetask/combinertest/CombinerTest.java | 9 +-
.../combinertest/LargeKVCombinerTest.java | 11 +-
.../combinertest/OldAPICombinerTest.java | 21 +-
.../nativetask/combinertest/WordCount.java | 3 +-
.../combinertest/WordCountWithOldAPI.java | 9 +-
.../nativetask/compresstest/CompressMapper.java | 6 +-
.../nativetask/handlers/TestCombineHandler.java | 7 +-
.../TestNativeCollectorOnlyHandler.java | 15 +-
.../nativetask/kvtest/HashSumReducer.java | 3 +-
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 17 +-
.../mapred/nativetask/kvtest/TestInputFile.java | 3 +-
.../nativetask/nonsorttest/NonSortTest.java | 14 +-
.../nativetask/nonsorttest/NonSortTestMR.java | 3 +-
.../nativetask/serde/TestKVSerializer.java | 18 +-
.../serde/TestNativeSerialization.java | 3 +-
.../EnforceNativeOutputCollectorDelegator.java | 3 +-
.../nativetask/testutil/ResultVerifier.java | 15 +-
.../mapred/nativetask/testutil/TestInput.java | 9 +-
136 files changed, 546 insertions(+), 2335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 279b960..ea17907 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -21,3 +21,4 @@ MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is suces
MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd)
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
+MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
index 612fdf9..0780741 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE
@@ -204,28 +204,6 @@
Licenses for third party projects used by this project:
-CityHash src/main/native/cityhash
----------------------------------------------------------------------
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
GoogleTest src/main/native/gtest
---------------------------------------------------------------------
Copyright 2008, Google Inc.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 0ab99db..7eed176 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -178,7 +178,6 @@ configure_file(main/native/test.sh test/test.sh)
add_dual_library(nativetask
${CMAKE_BINARY_DIR}/lz4.c
- ${D}/cityhash/city.cc
${D}/src/codec/BlockCodec.cc
${D}/src/codec/GzipCodec.cc
${D}/src/codec/Lz4Codec.cc
@@ -208,10 +207,8 @@ add_dual_library(nativetask
${D}/src/lib/SpillInfo.cc
${D}/src/lib/Path.cc
${D}/src/lib/Streams.cc
- ${D}/src/lib/Combiner.cc
${D}/src/lib/TaskCounters.cc
${D}/src/util/Checksum.cc
- ${D}/src/util/Hash.cc
${D}/src/util/Random.cc
${D}/src/util/StringUtil.cc
${D}/src/util/SyncUtils.cc
@@ -237,11 +234,8 @@ add_executable(nttest
${D}/test/lib/TestPartitionBucket.cc
${D}/test/lib/TestReadBuffer.cc
${D}/test/lib/TestReadWriteBuffer.cc
- ${D}/test/lib/TestTrackingCollector.cc
${D}/test/util/TestChecksum.cc
- ${D}/test/util/TestHash.cc
${D}/test/util/TestStringUtil.cc
- ${D}/test/util/TestSyncUtils.cc
${D}/test/util/TestWritableUtils.cc
${D}/test/TestCommand.cc
${D}/test/TestConfig.cc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
index ff1a95c..1f7a5d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java
@@ -25,22 +25,9 @@ public class Constants {
public static final String MAP_SORT_CLASS = "map.sort.class";
public static final String MAPRED_COMBINER_CLASS = "mapred.combiner.class";
- public static final String MAPRED_MAPTASK_DELEGATOR_CLASS = "mapreduce.map.task.delegator.class";
- public static final String MAPRED_REDUCETASK_DELEGATOR_CLASS = "mapreduce.reduce.task.delegator.class";
- public static final String NATIVE_TASK_ENABLED = "native.task.enabled";
public static final String NATIVE_LOG_DEVICE = "native.log.device";
public static final String NATIVE_HADOOP_VERSION = "native.hadoop.version";
- public static final String NATIVE_MAPPER_CLASS = "native.mapper.class";
- public static final String NATIVE_REDUCER_CLASS = "native.reducer.class";
- public static final String NATIVE_PARTITIONER_CLASS = "native.partitioner.class";
- public static final String NATIVE_COMBINER_CLASS = "native.combiner.class";
- public static final String NATIVE_INPUT_SPLIT = "native.input.split";
-
- public static final String NATIVE_RECORDREADER_CLASS = "native.recordreader.class";
- public static final String NATIVE_RECORDWRITER_CLASS = "native.recordwriter.class";
- public static final String NATIVE_OUTPUT_FILE_NAME = "native.output.file.name";
-
public static final String NATIVE_PROCESSOR_BUFFER_KB = "native.processor.buffer.kb";
public static final int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64;
public static final int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
index 85f78ca..b3bdf53 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java
@@ -32,9 +32,6 @@ public interface DataReceiver {
/**
* Send a signal to the receiver that the data arrives.
* The data is transferred in another band.
- *
- * @return
- * @throws IOException
*/
public boolean receiveData() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
index e4e706b..d28b883c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java
@@ -29,7 +29,6 @@ public interface ICombineHandler {
/**
* run combiner
- * @throws IOException
*/
public void combine() throws IOException;
@@ -40,7 +39,6 @@ public interface ICombineHandler {
/**
* close handlers, buffer pullers and pushers
- * @throws IOException
*/
public void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
index 9506c7f..fef56d9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java
@@ -45,17 +45,9 @@ public interface INativeHandler extends NativeDataTarget, NativeDataSource {
/**
* call command to downstream
- *
- * @param command
- * @param parameter
- * @return
- * @throws IOException
*/
public ReadWriteBuffer call(Command command, ReadWriteBuffer parameter) throws IOException;
- /**
- * @param handler
- */
void setCommandDispatcher(CommandDispatcher handler);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index 3c3811f..9458c6b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -229,32 +229,21 @@ public class NativeBatchProcessor implements INativeHandler {
/**
* Let native side to process data in inputBuffer
- *
- * @param handler
- * @param length
*/
private native void nativeProcessInput(long handler, int length);
/**
* Notice native side input is finished
- *
- * @param handler
*/
private native void nativeFinish(long handler);
/**
* Send control message to native side
- *
- * @param cmd
- * command data
- * @return return value
*/
private native byte[] nativeCommand(long handler, int cmd, byte[] parameter);
/**
* Load data from native
- *
- * @return
*/
private native void nativeLoadData(long handler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 828d7df..5a516c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -95,7 +95,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) {
- String message = "Native-Task doesn't support sort class " + job.get(Constants.MAP_SORT_CLASS);
+ String message = "Native-Task doesn't support sort class " +
+ job.get(Constants.MAP_SORT_CLASS);
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -115,8 +116,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
LOG.error(message);
throw new InvalidJobConfException(message);
} else if (!Platforms.support(keyCls.getName(), serializer, job)) {
- String message = "Native output collector doesn't support this key, this key is not comparable in native "
- + keyCls.getName();
+ String message = "Native output collector doesn't support this key, " +
+ "this key is not comparable in native: " + keyCls.getName();
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -144,7 +145,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect
updater.start();
} else {
- String message = "Nativeruntime cannot be loaded, please check the libnativetask.so is in hadoop library dir";
+ String message = "NativeRuntime cannot be loaded, please check that " +
+ "libnativetask.so is in hadoop library dir";
LOG.error(message);
throw new InvalidJobConfException(message);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index 395cc89..a0e88bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -79,9 +79,6 @@ public class NativeRuntime {
/**
* create native object We use it to create native handlers
- *
- * @param clazz
- * @return
*/
public synchronized static long createNativeObject(String clazz) {
assertNativeLibraryLoaded();
@@ -94,13 +91,11 @@ public class NativeRuntime {
/**
* Register a customized library
- *
- * @param clazz
- * @return
*/
public synchronized static long registerLibrary(String libraryName, String clazz) {
assertNativeLibraryLoaded();
- final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8), clazz.getBytes(Charsets.UTF_8));
+ final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8),
+ clazz.getBytes(Charsets.UTF_8));
if (ret != 0) {
LOG.warn("Can't create NativeObject for class " + clazz + ", probably not exist.");
}
@@ -117,9 +112,6 @@ public class NativeRuntime {
/**
* Get the status report from native space
- *
- * @param reporter
- * @throws IOException
*/
public static void reportStatus(TaskReporter reporter) throws IOException {
assertNativeLibraryLoaded();
@@ -164,40 +156,32 @@ public class NativeRuntime {
/**
* Config the native runtime with mapreduce job configurations.
- *
- * @param configs
*/
private native static void JNIConfigure(byte[][] configs);
/**
* create a native object in native space
- *
- * @param clazz
- * @return
*/
private native static long JNICreateNativeObject(byte[] clazz);
/**
* create the default native object for certain type
- *
- * @param type
- * @return
*/
@Deprecated
private native static long JNICreateDefaultNativeObject(byte[] type);
/**
* destroy native object in native space
- *
- * @param addr
*/
private native static void JNIReleaseNativeObject(long addr);
/**
- * get status update from native side Encoding: progress:float status:Text Counter number: int the count of the
- * counters Counters: array [group:Text, name:Text, incrCount:Long]
- *
- * @return
+ * Get status update from native side
+ * Encoding:
+ * progress:float
+ * status:Text
+ * number: int the count of the counters
+ * Counters: array [group:Text, name:Text, incrCount:Long]
*/
private native static byte[] JNIUpdateStatus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
index 7e41989..efec26c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java
@@ -49,8 +49,6 @@ public abstract class Platform {
/**
* initialize a platform, where we should call registerKey
- *
- * @throws IOException
*/
public abstract void init() throws IOException;
@@ -65,7 +63,6 @@ public abstract class Platform {
*
* @param keyClassName map out key class name
* @param key key serializer class
- * @throws IOException
*/
protected void registerKey(String keyClassName, Class<?> key) throws IOException {
serialization.register(keyClassName, key);
@@ -92,12 +89,12 @@ public abstract class Platform {
/**
* whether it's the platform that has defined a custom Java comparator
*
- * NativeTask doesn't support custom Java comparator(set with mapreduce.job.output.key.comparator.class)
- * but a platform (e.g Pig) could also set that conf and implement native comparators so
- * we shouldn't bail out.
+ * NativeTask doesn't support custom Java comparators
+ * (set with mapreduce.job.output.key.comparator.class)
+ * but a platform (e.g Pig) could also set that conf and implement native
+ * comparators so we shouldn't bail out.
*
* @param keyComparator comparator set with mapreduce.job.output.key.comparator.class
- * @return
*/
protected abstract boolean define(Class<?> keyComparator);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
index 1047bbf..9fd7a04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java
@@ -46,7 +46,8 @@ public class BufferPullee<IK, IV> implements IDataLoader {
private final NativeDataTarget target;
private boolean closed = false;
- public BufferPullee(Class<IK> iKClass, Class<IV> iVClass, RawKeyValueIterator rIter, NativeDataTarget target)
+ public BufferPullee(Class<IK> iKClass, Class<IV> iVClass,
+ RawKeyValueIterator rIter, NativeDataTarget target)
throws IOException {
this.rIter = rIter;
tmpInputKey = new SizedWritable<IK>(iKClass);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
index 18523ef..b5dff92 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
@@ -189,8 +189,6 @@ public class BufferPuller implements RawKeyValueIterator, DataReceiver {
/**
* Closes the iterator so that the underlying streams can be closed.
- *
- * @throws IOException
*/
@Override
public void close() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
index fe78325..090882d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
@@ -51,7 +51,8 @@ public class BufferPushee<OK, OV> implements Closeable {
private KVSerializer<OK, OV> deserializer;
private boolean closed = false;
- public BufferPushee(Class<OK> oKClass, Class<OV> oVClass, RecordWriter<OK, OV> writer) throws IOException {
+ public BufferPushee(Class<OK> oKClass, Class<OV> oVClass,
+ RecordWriter<OK, OV> writer) throws IOException {
tmpOutputKey = new SizedWritable<OK>(oKClass);
tmpOutputValue = new SizedWritable<OV>(oVClass);
@@ -128,7 +129,8 @@ public class BufferPushee<OK, OV> implements Closeable {
}
}
if (remain != totalRead) {
- throw new IOException("We expect to read " + remain + ", but we actually read: " + totalRead);
+ throw new IOException("We expect to read " + remain +
+ ", but we actually read: " + totalRead);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
index 10d619b..454127c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
@@ -44,7 +44,8 @@ public class BufferPusher<K, V> implements OutputCollector<K, V> {
IKVSerializer serializer;
private boolean closed = false;
- public BufferPusher(Class<K> iKClass, Class<V> iVClass, NativeDataTarget target) throws IOException {
+ public BufferPusher(Class<K> iKClass, Class<V> iVClass,
+ NativeDataTarget target) throws IOException {
tmpInputKey = new SizedWritable<K>(iKClass);
tmpInputValue = new SizedWritable<V>(iVClass);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
index dad7af9..fcaeca1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
@@ -49,7 +49,8 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
private final BufferPusher<K, V> kvPusher;
private boolean closed = false;
- public static <K, V> ICombineHandler create(TaskContext context) throws IOException, ClassNotFoundException {
+ public static <K, V> ICombineHandler create(TaskContext context)
+ throws IOException, ClassNotFoundException {
final JobConf conf = new JobConf(context.getConf());
conf.set(Constants.SERIALIZATION_FRAMEWORK,
String.valueOf(SerializationFramework.WRITABLE_SERIALIZATION.getType()));
@@ -66,11 +67,13 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
final Counter combineInputCounter = context.getTaskReporter().getCounter(
TaskCounter.COMBINE_INPUT_RECORDS);
-
- final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(conf, context.getTaskAttemptId(),
+
+ final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(
+ conf, context.getTaskAttemptId(),
combineInputCounter, context.getTaskReporter(), null);
- final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, conf, DataChannel.INOUT);
+ final INativeHandler nativeHandler = NativeBatchProcessor.create(
+ NAME, conf, DataChannel.INOUT);
@SuppressWarnings("unchecked")
final BufferPusher<K, V> pusher = new BufferPusher<K, V>((Class<K>)context.getInputKeyClass(),
(Class<V>)context.getInputValueClass(),
@@ -79,8 +82,9 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
return new CombinerHandler<K, V>(nativeHandler, combinerRunner, puller, pusher);
}
- public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner, BufferPuller puller,
- BufferPusher<K, V> kvPusher) throws IOException {
+ public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner,
+ BufferPuller puller, BufferPusher<K, V> kvPusher)
+ throws IOException {
this.nativeHandler = nativeHandler;
this.combinerRunner = combiner;
this.puller = puller;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
index d2cbf1f..6eaf28b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java
@@ -30,7 +30,6 @@ public interface IDataLoader {
/**
* @return size of data loaded
- * @throws IOException
*/
public int load() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
index 284c3dc..34238ae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
@@ -63,7 +63,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
private final INativeHandler nativeHandler;
private boolean closed = false;
- public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context) throws IOException {
+ public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context)
+ throws IOException {
ICombineHandler combinerHandler = null;
@@ -81,7 +82,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos
LOG.info("[NativeCollectorOnlyHandler] combiner is not null");
}
- final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, context.getConf(), DataChannel.OUT);
+ final INativeHandler nativeHandler = NativeBatchProcessor.create(
+ NAME, context.getConf(), DataChannel.OUT);
final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>(
(Class<K>)context.getOutputKeyClass(),
(Class<V>)context.getOutputValueClass(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
index 08264f9..48dccfa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
@InterfaceAudience.Private
-public class BytesWritableSerializer implements INativeComparable, INativeSerializer<BytesWritable> {
+public class BytesWritableSerializer
+ implements INativeComparable, INativeSerializer<BytesWritable> {
@Override
public int getLength(BytesWritable w) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
index 8506d17..15a2ef7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java
@@ -33,43 +33,16 @@ public interface IKVSerializer {
/**
* update the length field of SizedWritable
- * @param key
- * @param value
- * @throws IOException
*/
public void updateLength(SizedWritable<?> key, SizedWritable<?> value) throws IOException;
- /**
- *
- * @param out
- * @param key
- * @param value
- * @return bytes written
- * @throws IOException
- */
public int serializeKV(DataOutputStream out, SizedWritable<?> key,
SizedWritable<?> value) throws IOException;
- /**
- * serialize partitionId as well
- * @param out
- * @param partitionId
- * @param key
- * @param value
- * @return
- * @throws IOException
- */
public int serializePartitionKV(DataOutputStream out, int partitionId,
SizedWritable<?> key, SizedWritable<?> value)
throws IOException;
- /**
- *
- * @param in
- * @param key
- * @param value
- * @return bytes read
- * @throws IOException
- */
- public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value) throws IOException;
+ public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
index d650a4d..e6609b8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
@@ -55,7 +55,8 @@ public class KVSerializer<K, V> implements IKVSerializer {
}
@Override
- public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) throws IOException {
+ public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value)
+ throws IOException {
return serializePartitionKV(out, -1, key, value);
}
@@ -64,7 +65,8 @@ public class KVSerializer<K, V> implements IKVSerializer {
SizedWritable<?> key, SizedWritable<?> value)
throws IOException {
- if (key.length == SizedWritable.INVALID_LENGTH || value.length == SizedWritable.INVALID_LENGTH) {
+ if (key.length == SizedWritable.INVALID_LENGTH ||
+ value.length == SizedWritable.INVALID_LENGTH) {
updateLength(key, value);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
index 5881a46..11bde64 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.io.Writable;
@InterfaceAudience.Private
public class NativeSerialization {
- private final ConcurrentHashMap<String, Class<?>> map = new ConcurrentHashMap<String, Class<?>>();
+ private final ConcurrentHashMap<String, Class<?>> map =
+ new ConcurrentHashMap<String, Class<?>>();
public boolean accept(Class<?> c) {
return Writable.class.isAssignableFrom(c);
@@ -40,7 +41,8 @@ public class NativeSerialization {
return null;
}
if (!Writable.class.isAssignableFrom(c)) {
- throw new IOException("Cannot serialize type " + c.getName() + ", we only accept subclass of Writable");
+ throw new IOException("Cannot serialize type " + c.getName() +
+ ", we only accept subclass of Writable");
}
final String name = c.getName();
final Class<?> serializer = map.get(name);
@@ -70,8 +72,9 @@ public class NativeSerialization {
return;
} else {
if (!storedSerializer.getName().equals(serializer.getName())) {
- throw new IOException("Error! Serializer already registered, exist: " + storedSerializer.getName() + ", new: "
- + serializer.getName());
+ throw new IOException("Error! Serializer already registered, existing: " +
+ storedSerializer.getName() + ", new: " +
+ serializer.getName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index 35ef0a9..16f6919 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -97,7 +97,7 @@ public class BytesUtil {
* Write a printable representation of a byte array.
*
* @param b byte array
- * @return string
+ * @return the printable presentation
* @see #toStringBinary(byte[], int, int)
*/
public static String toStringBinary(final byte [] b) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
index 6d63be5..aa97789 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java
@@ -45,9 +45,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -57,10 +54,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map output file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -69,9 +63,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -81,10 +72,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT);
@@ -94,10 +82,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber);
@@ -107,12 +92,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber);
@@ -122,10 +103,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException {
String path = String
@@ -136,12 +114,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException {
String path = String
@@ -152,10 +126,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException {
return lDirAlloc.getLocalPathToRead(
@@ -166,14 +137,11 @@ public class LocalJobOutputFiles implements NativeTaskOutput {
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
- public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException {
+ public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf)
+ throws IOException {
return lDirAlloc.getLocalPathForWrite(
String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size,
conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
index e92901b..7c9f433 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java
@@ -32,103 +32,70 @@ public interface NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException;
/**
* Create a local map output file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException;
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException;
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException;
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException;
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException;
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException;
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ r* @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException;
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException;
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
index 5cad10b..d52ee1c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java
@@ -28,11 +28,13 @@ import org.apache.hadoop.mapred.TaskID;
/**
* Manipulate the working area for the transient store for maps and reduces.
- *
- * This class is used by map and reduce tasks to identify the directories that they need to write
- * to/read from for intermediate files. The callers of these methods are from child space and see
- * mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used
- * from TaskTracker space.
+ *
+ * This class is used by map and reduce tasks to identify the directories that they need
+ * to write to/read from for intermediate files. The callers of these methods are from
+ * child space and see mapreduce.cluster.local.dir as
+ * taskTracker/jobCache/jobId/attemptId.
+ *
+ * This class should not be used from TaskTracker space.
*/
@InterfaceAudience.Private
public class NativeTaskOutputFiles implements NativeTaskOutput {
@@ -55,9 +57,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return the path to local map output file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputFile() throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -66,11 +65,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map output file name.
- *
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ *
+ * @param size the size of the file
*/
public Path getOutputFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -79,9 +75,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return the path to a local map output index file created earlier
- *
- * @return path
- * @throws IOException
*/
public Path getOutputIndexFile() throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -91,10 +84,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map output index file name.
*
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param size the size of the file
*/
public Path getOutputIndexFileForWrite(long size) throws IOException {
String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id);
@@ -104,10 +94,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill file created earlier.
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillFile(int spillNumber) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber);
@@ -117,12 +104,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillFileForWrite(int spillNumber, long size) throws IOException {
String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber);
@@ -132,10 +115,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local map spill index file created earlier
*
- * @param spillNumber
- * the number
- * @return path
- * @throws IOException
+ * @param spillNumber the number
*/
public Path getSpillIndexFile(int spillNumber) throws IOException {
String path = String
@@ -146,12 +126,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local map spill index file name.
*
- * @param spillNumber
- * the number
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param spillNumber the number
+ * @param size the size of the file
*/
public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException {
String path = String
@@ -162,10 +138,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Return a local reduce input file created earlier
*
- * @param mapId
- * a map task id
- * @return path
- * @throws IOException
+ * @param mapId a map task id
*/
public Path getInputFile(int mapId) throws IOException {
return lDirAlloc.getLocalPathToRead(
@@ -176,14 +149,11 @@ public class NativeTaskOutputFiles implements NativeTaskOutput {
/**
* Create a local reduce input file name.
*
- * @param mapId
- * a map task id
- * @param size
- * the size of the file
- * @return path
- * @throws IOException
+ * @param mapId a map task id
+ * @param size the size of the file
*/
- public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException {
+ public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf)
+ throws IOException {
return lDirAlloc.getLocalPathForWrite(
String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size,
conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
index 017d834..03df016 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java
@@ -112,10 +112,15 @@ public class ReadWriteBuffer {
}
public long readLong() {
- final long result = ((_buff[_readPoint + 0] & 255) << 0) + ((_buff[_readPoint + 1] & 255) << 8)
- + ((_buff[_readPoint + 2] & 255) << 16) + ((long) (_buff[_readPoint + 3] & 255) << 24)
- + ((long) (_buff[_readPoint + 4] & 255) << 32) + ((long) (_buff[_readPoint + 5] & 255) << 40)
- + ((long) (_buff[_readPoint + 6] & 255) << 48) + (((long) _buff[_readPoint + 7] << 56));
+ final long result =
+ ((_buff[_readPoint + 0] & 255) << 0) +
+ ((_buff[_readPoint + 1] & 255) << 8) +
+ ((_buff[_readPoint + 2] & 255) << 16) +
+ ((long) (_buff[_readPoint + 3] & 255) << 24) +
+ ((long) (_buff[_readPoint + 4] & 255) << 32) +
+ ((long) (_buff[_readPoint + 5] & 255) << 40) +
+ ((long) (_buff[_readPoint + 6] & 255) << 48) +
+ (((long) _buff[_readPoint + 7] << 56));
_readPoint += 8;
return result;
@@ -144,7 +149,8 @@ public class ReadWriteBuffer {
if (_buff.length - _writePoint >= toBeWritten) {
return;
}
- final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ? (toBeWritten + _writePoint) : CACHE_LINE_SIZE;
+ final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ?
+ (toBeWritten + _writePoint) : CACHE_LINE_SIZE;
final byte[] newBuff = new byte[newLength];
System.arraycopy(_buff, 0, newBuff, 0, _writePoint);
_buff = newBuff;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
deleted file mode 100644
index 36ff93b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc
+++ /dev/null
@@ -1,307 +0,0 @@
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-//
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
-//
-// This file provides CityHash64() and related functions.
-//
-// It's probably possible to create even faster hash functions by
-// writing a program that systematically explores some of the space of
-// possible hash functions, by using SIMD instructions, or by
-// compromising on hash quality.
-
-#include "city.h"
-
-#include <algorithm>
-
-using namespace std;
-
-#define UNALIGNED_LOAD64(p) (*(const uint64*)(p))
-#define UNALIGNED_LOAD32(p) (*(const uint32*)(p))
-
-#if !defined(LIKELY)
-#if defined(__GNUC__)
-#define LIKELY(x) (__builtin_expect(!!(x), 1))
-#else
-#define LIKELY(x) (x)
-#endif
-#endif
-
-// Some primes between 2^63 and 2^64 for various uses.
-static const uint64 k0 = 0xc3a5c85c97cb3127ULL;
-static const uint64 k1 = 0xb492b66fbe98f273ULL;
-static const uint64 k2 = 0x9ae16a3b2f90404fULL;
-static const uint64 k3 = 0xc949d7c7509e6557ULL;
-
-// Bitwise right rotate. Normally this will compile to a single
-// instruction, especially if the shift is a manifest constant.
-static uint64 Rotate(uint64 val, int shift) {
- // Avoid shifting by 64: doing so yields an undefined result.
- return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
-}
-
-// Equivalent to Rotate(), but requires the second arg to be non-zero.
-// On x86-64, and probably others, it's possible for this to compile
-// to a single instruction if both args are already in registers.
-static uint64 RotateByAtLeast1(uint64 val, int shift) {
- return (val >> shift) | (val << (64 - shift));
-}
-
-static uint64 ShiftMix(uint64 val) {
- return val ^ (val >> 47);
-}
-
-static uint64 HashLen16(uint64 u, uint64 v) {
- return Hash128to64(uint128(u, v));
-}
-
-static uint64 HashLen0to16(const char *s, size_t len) {
- if (len > 8) {
- uint64 a = UNALIGNED_LOAD64(s);
- uint64 b = UNALIGNED_LOAD64(s + len - 8);
- return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b;
- }
- if (len >= 4) {
- uint64 a = UNALIGNED_LOAD32(s);
- return HashLen16(len + (a << 3), UNALIGNED_LOAD32(s + len - 4));
- }
- if (len > 0) {
- uint8 a = s[0];
- uint8 b = s[len >> 1];
- uint8 c = s[len - 1];
- uint32 y = static_cast<uint32>(a) + (static_cast<uint32>(b) << 8);
- uint32 z = len + (static_cast<uint32>(c) << 2);
- return ShiftMix(y * k2 ^ z * k3) * k2;
- }
- return k2;
-}
-
-// This probably works well for 16-byte strings as well, but it may be overkill
-// in that case.
-static uint64 HashLen17to32(const char *s, size_t len) {
- uint64 a = UNALIGNED_LOAD64(s) * k1;
- uint64 b = UNALIGNED_LOAD64(s + 8);
- uint64 c = UNALIGNED_LOAD64(s + len - 8) * k2;
- uint64 d = UNALIGNED_LOAD64(s + len - 16) * k0;
- return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d,
- a + Rotate(b ^ k3, 20) - c + len);
-}
-
-// Return a 16-byte hash for 48 bytes. Quick and dirty.
-// Callers do best to use "random-looking" values for a and b.
-static pair<uint64, uint64> WeakHashLen32WithSeeds(
- uint64 w, uint64 x, uint64 y, uint64 z, uint64 a, uint64 b) {
- a += w;
- b = Rotate(b + a + z, 21);
- uint64 c = a;
- a += x;
- a += y;
- b += Rotate(a, 44);
- return make_pair(a + z, b + c);
-}
-
-// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty.
-static pair<uint64, uint64> WeakHashLen32WithSeeds(
- const char* s, uint64 a, uint64 b) {
- return WeakHashLen32WithSeeds(UNALIGNED_LOAD64(s),
- UNALIGNED_LOAD64(s + 8),
- UNALIGNED_LOAD64(s + 16),
- UNALIGNED_LOAD64(s + 24),
- a,
- b);
-}
-
-// Return an 8-byte hash for 33 to 64 bytes.
-static uint64 HashLen33to64(const char *s, size_t len) {
- uint64 z = UNALIGNED_LOAD64(s + 24);
- uint64 a = UNALIGNED_LOAD64(s) + (len + UNALIGNED_LOAD64(s + len - 16)) * k0;
- uint64 b = Rotate(a + z, 52);
- uint64 c = Rotate(a, 37);
- a += UNALIGNED_LOAD64(s + 8);
- c += Rotate(a, 7);
- a += UNALIGNED_LOAD64(s + 16);
- uint64 vf = a + z;
- uint64 vs = b + Rotate(a, 31) + c;
- a = UNALIGNED_LOAD64(s + 16) + UNALIGNED_LOAD64(s + len - 32);
- z = UNALIGNED_LOAD64(s + len - 8);
- b = Rotate(a + z, 52);
- c = Rotate(a, 37);
- a += UNALIGNED_LOAD64(s + len - 24);
- c += Rotate(a, 7);
- a += UNALIGNED_LOAD64(s + len - 16);
- uint64 wf = a + z;
- uint64 ws = b + Rotate(a, 31) + c;
- uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0);
- return ShiftMix(r * k0 + vs) * k2;
-}
-
-uint64 CityHash64(const char *s, size_t len) {
- if (len <= 32) {
- if (len <= 16) {
- return HashLen0to16(s, len);
- } else {
- return HashLen17to32(s, len);
- }
- } else if (len <= 64) {
- return HashLen33to64(s, len);
- }
-
- // For strings over 64 bytes we hash the end first, and then as we
- // loop we keep 56 bytes of state: v, w, x, y, and z.
- uint64 x = UNALIGNED_LOAD64(s);
- uint64 y = UNALIGNED_LOAD64(s + len - 16) ^ k1;
- uint64 z = UNALIGNED_LOAD64(s + len - 56) ^ k0;
- pair<uint64, uint64> v = WeakHashLen32WithSeeds(s + len - 64, len, y);
- pair<uint64, uint64> w = WeakHashLen32WithSeeds(s + len - 32, len * k1, k0);
- z += ShiftMix(v.second) * k1;
- x = Rotate(z + x, 39) * k1;
- y = Rotate(y, 33) * k1;
-
- // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
- len = (len - 1) & ~static_cast<size_t>(63);
- do {
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- len -= 64;
- } while (len != 0);
- return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z,
- HashLen16(v.second, w.second) + x);
-}
-
-uint64 CityHash64WithSeed(const char *s, size_t len, uint64 seed) {
- return CityHash64WithSeeds(s, len, k2, seed);
-}
-
-uint64 CityHash64WithSeeds(const char *s, size_t len,
- uint64 seed0, uint64 seed1) {
- return HashLen16(CityHash64(s, len) - seed0, seed1);
-}
-
-// A subroutine for CityHash128(). Returns a decent 128-bit hash for strings
-// of any length representable in ssize_t. Based on City and Murmur.
-static uint128 CityMurmur(const char *s, size_t len, uint128 seed) {
- uint64 a = Uint128Low64(seed);
- uint64 b = Uint128High64(seed);
- uint64 c = 0;
- uint64 d = 0;
- ssize_t l = len - 16;
- if (l <= 0) { // len <= 16
- c = b * k1 + HashLen0to16(s, len);
- d = Rotate(a + (len >= 8 ? UNALIGNED_LOAD64(s) : c), 32);
- } else { // len > 16
- c = HashLen16(UNALIGNED_LOAD64(s + len - 8) + k1, a);
- d = HashLen16(b + len, c + UNALIGNED_LOAD64(s + len - 16));
- a += d;
- do {
- a ^= ShiftMix(UNALIGNED_LOAD64(s) * k1) * k1;
- a *= k1;
- b ^= a;
- c ^= ShiftMix(UNALIGNED_LOAD64(s + 8) * k1) * k1;
- c *= k1;
- d ^= c;
- s += 16;
- l -= 16;
- } while (l > 0);
- }
- a = HashLen16(a, c);
- b = HashLen16(d, b);
- return uint128(a ^ b, HashLen16(b, a));
-}
-
-uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed) {
- if (len < 128) {
- return CityMurmur(s, len, seed);
- }
-
- // We expect len >= 128 to be the common case. Keep 56 bytes of state:
- // v, w, x, y, and z.
- pair<uint64, uint64> v, w;
- uint64 x = Uint128Low64(seed);
- uint64 y = Uint128High64(seed);
- uint64 z = len * k1;
- v.first = Rotate(y ^ k1, 49) * k1 + UNALIGNED_LOAD64(s);
- v.second = Rotate(v.first, 42) * k1 + UNALIGNED_LOAD64(s + 8);
- w.first = Rotate(y + z, 35) * k1 + x;
- w.second = Rotate(x + UNALIGNED_LOAD64(s + 88), 53) * k1;
-
- // This is the same inner loop as CityHash64(), manually unrolled.
- do {
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
- y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
- x ^= w.second;
- y ^= v.first;
- z = Rotate(z ^ w.first, 33);
- v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
- w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
- std::swap(z, x);
- s += 64;
- len -= 128;
- } while (LIKELY(len >= 128));
- y += Rotate(w.first, 37) * k0 + z;
- x += Rotate(v.first + z, 49) * k0;
- // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
- for (size_t tail_done = 0; tail_done < len; ) {
- tail_done += 32;
- y = Rotate(y - x, 42) * k0 + v.second;
- w.first += UNALIGNED_LOAD64(s + len - tail_done + 16);
- x = Rotate(x, 49) * k0 + w.first;
- w.first += v.first;
- v = WeakHashLen32WithSeeds(s + len - tail_done, v.first, v.second);
- }
- // At this point our 48 bytes of state should contain more than
- // enough information for a strong 128-bit hash. We use two
- // different 48-byte-to-8-byte hashes to get a 16-byte final result.
- x = HashLen16(x, v.first);
- y = HashLen16(y, w.first);
- return uint128(HashLen16(x + v.second, w.second) + y,
- HashLen16(x + w.second, y + v.second));
-}
-
-uint128 CityHash128(const char *s, size_t len) {
- if (len >= 16) {
- return CityHash128WithSeed(s + 16,
- len - 16,
- uint128(UNALIGNED_LOAD64(s) ^ k3,
- UNALIGNED_LOAD64(s + 8)));
- } else if (len >= 8) {
- return CityHash128WithSeed(NULL,
- 0,
- uint128(UNALIGNED_LOAD64(s) ^ (len * k0),
- UNALIGNED_LOAD64(s + len - 8) ^ k1));
- } else {
- return CityHash128WithSeed(s, len, uint128(k0, k1));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
deleted file mode 100644
index 7b290be..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright (c) 2011 Google, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-//
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
-//
-// This file provides a few functions for hashing strings. On x86-64
-// hardware in 2011, CityHash64() is faster than other high-quality
-// hash functions, such as Murmur. This is largely due to higher
-// instruction-level parallelism. CityHash64() and CityHash128() also perform
-// well on hash-quality tests.
-//
-// CityHash128() is optimized for relatively long strings and returns
-// a 128-bit hash. For strings more than about 2000 bytes it can be
-// faster than CityHash64().
-//
-// Functions in the CityHash family are not suitable for cryptography.
-//
-// WARNING: This code has not been tested on big-endian platforms!
-// It is known to work well on little-endian platforms that have a small penalty
-// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs.
-//
-// By the way, for some hash functions, given strings a and b, the hash
-// of a+b is easily derived from the hashes of a and b. This property
-// doesn't hold for any hash functions in this file.
-
-#ifndef CITY_HASH_H_
-#define CITY_HASH_H_
-
-#include <stdlib.h> // for size_t.
-#include <stdint.h>
-#include <utility>
-
-typedef uint8_t uint8;
-typedef uint32_t uint32;
-typedef uint64_t uint64;
-typedef std::pair<uint64, uint64> uint128;
-
-inline uint64 Uint128Low64(const uint128& x) { return x.first; }
-inline uint64 Uint128High64(const uint128& x) { return x.second; }
-
-// Hash function for a byte array.
-uint64 CityHash64(const char *buf, size_t len);
-
-// Hash function for a byte array. For convenience, a 64-bit seed is also
-// hashed into the result.
-uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed);
-
-// Hash function for a byte array. For convenience, two seeds are also
-// hashed into the result.
-uint64 CityHash64WithSeeds(const char *buf, size_t len,
- uint64 seed0, uint64 seed1);
-
-// Hash function for a byte array.
-uint128 CityHash128(const char *s, size_t len);
-
-// Hash function for a byte array. For convenience, a 128-bit seed is also
-// hashed into the result.
-uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed);
-
-// Hash 128 input bits down to 64 bits of output.
-// This is intended to be a reasonably good hash function.
-inline uint64 Hash128to64(const uint128& x) {
- // Murmur-inspired hashing.
- const uint64 kMul = 0x9ddfea08eb382d69ULL;
- uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul;
- a ^= (a >> 47);
- uint64 b = (Uint128High64(x) ^ a) * kMul;
- b ^= (b >> 47);
- b *= kMul;
- return b;
-}
-
-#endif // CITY_HASH_H_