You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/13 05:54:43 UTC
[37/50] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of
unused code. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 60bb6f5..ba026f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -38,13 +38,6 @@ using std::pair;
enum NativeObjectType {
UnknownObjectType = 0,
BatchHandlerType = 1,
- MapperType = 2,
- ReducerType = 3,
- PartitionerType = 4,
- CombinerType = 5,
- FolderType = 6,
- RecordReaderType = 7,
- RecordWriterType = 8
};
/**
@@ -69,7 +62,6 @@ enum Endium {
#define INPUT_LINE_KV_SEPERATOR "mapreduce.input.keyvaluelinerecordreader.key.value.separator"
#define MAPRED_TEXTOUTPUT_FORMAT_SEPERATOR "mapreduce.output.textoutputformat.separator"
#define MAPRED_WORK_OUT_DIR "mapreduce.task.output.dir"
-#define NATIVE_OUTPUT_FILE_NAME "native.output.file.name"
#define MAPRED_COMPRESS_OUTPUT "mapreduce.output.fileoutputformat.compress"
#define MAPRED_OUTPUT_COMPRESSION_CODEC "mapreduce.output.fileoutputformat.compress.codec"
#define TOTAL_ORDER_PARTITIONER_PATH "total.order.partitioner.path"
@@ -386,32 +378,6 @@ public:
virtual bool next(Buffer & key, Buffer & value) = 0;
};
-class RecordReader : public KVIterator, public Configurable, public Progress {
-public:
- virtual NativeObjectType type() {
- return RecordReaderType;
- }
-
- virtual bool next(Buffer & key, Buffer & value) = 0;
-
- virtual float getProgress() = 0;
-
- virtual void close() = 0;
-};
-
-class RecordWriter : public Collector, public Configurable {
-public:
- virtual NativeObjectType type() {
- return RecordWriterType;
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) {
- }
-
- virtual void close() {
- }
-
-};
class ProcessorBase : public Configurable {
protected:
@@ -444,36 +410,6 @@ public:
}
};
-class Mapper : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return MapperType;
- }
-
- /**
- * Map interface, default IdenticalMapper
- */
- virtual void map(const char * key, uint32_t keyLen, const char * value, uint32_t valueLen) {
- collect(key, keyLen, value, valueLen);
- }
-};
-
-class Partitioner : public Configurable {
-public:
- virtual NativeObjectType type() {
- return PartitionerType;
- }
-
- /**
- * Partition interface
- * @param key key buffer
- * @param keyLen key length, can be modified to smaller value
- * to truncate key
- * @return partition number
- */
- virtual uint32_t getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition);
-};
-
enum KeyGroupIterState {
SAME_KEY,
NEW_KEY,
@@ -502,80 +438,7 @@ public:
virtual const char * nextValue(uint32_t & len) = 0;
};
-class Reducer : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return ReducerType;
- }
-
- /**
- * Reduce interface, default IdenticalReducer
- */
- virtual void reduce(KeyGroupIterator & input) {
- const char * key;
- const char * value;
- uint32_t keyLen;
- uint32_t valueLen;
- key = input.getKey(keyLen);
- while (NULL != (value = input.nextValue(valueLen))) {
- collect(key, keyLen, value, valueLen);
- }
- }
-};
-
-/**
- * Folder API used for hashtable based aggregation
- * Folder will be used in this way:
- * on(key, value):
- * state = hashtable.get(key)
- * if state == None:
- * size = size()
- * if size == -1:
- * state = init(null, -1)
- * elif size > 0:
- * state = fixallocator.get(key)
- * init(state, size)
- * folder(state, value, value.len)
- *
- * final():
- * for k,state in hashtable:
- * final(key, key.len, state)
- */
-class Folder : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
- return FolderType;
- }
-
- /**
- * Get aggregator state size
- * @return state storage size
- * -1 size not fixed or unknown, default
- * e.g. list map tree
- * 0 don't need to store state
- * >0 fixed sized state
- * e.g. int32 int64 float.
- */
- virtual int32_t size() {
- return -1;
- }
- /**
- * Create and/or init new state
- */
- virtual void * init(const char * key, uint32_t keyLen) {
- return NULL;
- }
-
- /**
- * Aggregation function
- */
- virtual void folder(void * dest, const char * value, uint32_t valueLen) {
- }
-
- virtual void final(const char * key, uint32_t keyLen, void * dest) {
- }
-};
enum KeyValueType {
TextType = 0,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
index 59cd8fb..ce36239 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
#include "BlockCodec.h"
@@ -79,9 +79,8 @@ BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t buff
}
void BlockDecompressStream::init() {
- _tempBufferSize = maxCompressedLength(_blockMax) + 8;
+ _tempBufferSize = maxCompressedLength(_blockMax) + 8;
_tempBuffer = (char*)malloc(_tempBufferSize);
-
}
BlockDecompressStream::~BlockDecompressStream() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
index 0412bb0..ffa4417 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h
@@ -19,7 +19,7 @@
#ifndef BLOCKCODEC_H_
#define BLOCKCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
index bb699e9..b0b9852 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
@@ -18,7 +18,7 @@
#include <zconf.h>
#include <zlib.h>
-#include "commons.h"
+#include "lib/commons.h"
#include "GzipCodec.h"
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
index 96721e5..a5f986e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h
@@ -19,7 +19,7 @@
#ifndef GZIPCODEC_H_
#define GZIPCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
index 8dc4c64..75bfad1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "lz4.h"
#include "NativeTask.h"
#include "Lz4Codec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
index d300dda..f3dd168 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h
@@ -19,7 +19,7 @@
#ifndef LZ4CODEC_H_
#define LZ4CODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
#include "BlockCodec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
index 0504033..a0417e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
@@ -19,7 +19,7 @@
#include "config.h"
#if defined HADOOP_SNAPPY_LIBRARY
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
#include "SnappyCodec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
index 3ba06b8..0673cf3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h
@@ -19,7 +19,7 @@
#ifndef SNAPPYCODEC_H_
#define SNAPPYCODEC_H_
-#include "Compressions.h"
+#include "lib/Compressions.h"
#include "BlockCodec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
index 7ed363b..37e0d42 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc
@@ -16,20 +16,19 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "MCollectorOutputHandler.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
#include "CombineHandler.h"
using std::string;
using std::vector;
namespace NativeTask {
-
const Command AbstractMapHandler::GET_OUTPUT_PATH(100, "GET_OUTPUT_PATH");
const Command AbstractMapHandler::GET_OUTPUT_INDEX_PATH(101, "GET_OUTPUT_INDEX_PATH");
const Command AbstractMapHandler::GET_SPILL_PATH(102, "GET_SPILL_PATH");
const Command AbstractMapHandler::GET_COMBINE_HANDLER(103, "GET_COMBINE_HANDLER");
-} //namespace
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
index b5b7451..ecb217c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc
@@ -19,11 +19,11 @@
#ifndef QUICK_BUILD
#include "org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h"
#endif
-#include "commons.h"
+#include "lib/commons.h"
#include "jni_md.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
#include "BatchHandler.h"
-#include "NativeObjectFactory.h"
+#include "lib/NativeObjectFactory.h"
///////////////////////////////////////////////////////////////
// NativeBatchProcessor jni util methods
@@ -130,7 +130,8 @@ void BatchHandler::onSetup(Config * config, char * inputBuffer, uint32_t inputBu
_out.reset(outputBuffer, outputBufferCapacity);
_out.rewind(0, outputBufferCapacity);
- LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d", inputBufferCapacity, _out.limit());
+ LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d",
+ inputBufferCapacity, _out.limit());
}
configure(_config);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
index 83da238..5f3863e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
@@ -208,7 +208,6 @@ void CombineHandler::write(char * buf, uint32_t length) {
outputRecordCount++;
remain -= kv->length();
pos += kv->length();
- ;
}
_combineOutputRecordCount += outputRecordCount;
@@ -242,7 +241,8 @@ void CombineHandler::combine(CombineContext type, KVIterator * kvIterator, IFile
this->_writer = writer;
call(COMBINE, NULL);
- LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, output Record Count: %d, output Bytes: %d",
+ LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, "
+ "output Record Count: %d, output Bytes: %d",
_combineInputRecordCount, _combineInputBytes,
_combineOutputRecordCount, _combineOutputBytes);
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
index 7f10552..eb43bd4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h
@@ -18,7 +18,7 @@
#ifndef _COMBINEHANDLER_H_
#define _COMBINEHANDLER_H_
-#include "Combiner.h"
+#include "lib/Combiner.h"
#include "BatchHandler.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
index 4df5e64..7e4ae44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "lib/TaskCounters.h"
#include "MCollectorOutputHandler.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
#include "CombineHandler.h"
using std::string;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
index 33a1de9..e459b46 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc
@@ -16,123 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "BufferStream.h"
+#include "lib/commons.h"
+#include "lib/BufferStream.h"
namespace NativeTask {
-BufferedInputStream::BufferedInputStream(InputStream * stream, uint32_t bufferSize)
- : FilterInputStream(stream), _buff(NULL), _position(0), _limit(0), _capacity(0) {
- _buff = (char*)malloc(bufferSize);
- if (NULL != _buff) {
- LOG("[BuferStream] malloc failed when create BufferedInputStream with buffersize %u",
- bufferSize);
- _capacity = bufferSize;
- }
-}
-
-BufferedInputStream::~BufferedInputStream() {
- if (NULL != _buff) {
- free(_buff);
- _buff = NULL;
- _position = 0;
- _limit = 0;
- _capacity = 0;
- }
-}
-
-void BufferedInputStream::seek(uint64_t position) {
- if (_limit - _position > 0) {
- THROW_EXCEPTION(IOException, "temporary buffered data exists when fseek()");
- }
- _stream->seek(position);
-}
-
-uint64_t BufferedInputStream::tell() {
- return _stream->tell() - (_limit - _position);
-}
-
-int32_t BufferedInputStream::read(void * buff, uint32_t length) {
- uint32_t rest = _limit - _position;
- if (rest > 0) {
- // have some data in buffer, read from buffer
- uint32_t cp = rest < length ? rest : length;
- memcpy(buff, _buff + _position, cp);
- _position += cp;
- return cp;
- } else if (length >= _capacity / 2) {
- // dest buffer big enough, read to dest buffer directly
- return _stream->read(buff, length);
- } else {
- // read to buffer first, then copy part of it to dest
- _limit = 0;
- do {
- int32_t rd = _stream->read(_buff + _limit, _capacity - _limit);
- if (rd <= 0) {
- break;
- }
- } while (_limit < _capacity / 2);
- if (_limit == 0) {
- return -1;
- }
- uint32_t cp = _limit < length ? _limit : length;
- memcpy(buff, _buff, cp);
- _position = cp;
- return cp;
- }
-}
-
-/////////////////////////////////////////////////////////////////
-
-BufferedOutputStream::BufferedOutputStream(InputStream * stream, uint32_t bufferSize)
- : FilterOutputStream(_stream), _buff(NULL), _position(0), _capacity(0) {
- _buff = (char*)malloc(bufferSize + sizeof(uint64_t));
- if (NULL != _buff) {
- LOG("[BuferStream] malloc failed when create BufferedOutputStream with buffersize %u",
- bufferSize);
- _capacity = bufferSize;
- }
-}
-
-BufferedOutputStream::~BufferedOutputStream() {
- if (NULL != _buff) {
- free(_buff);
- _buff = NULL;
- _position = 0;
- _capacity = 0;
- }
-}
-
-uint64_t BufferedOutputStream::tell() {
- return _stream->tell() + _position;
-}
-
-void BufferedOutputStream::write(const void * buff, uint32_t length) {
- if (length < _capacity / 2) {
- uint32_t rest = _capacity - _position;
- if (length < rest) {
- simple_memcpy(_buff + _position, buff, length);
- _position += length;
- } else {
- flush();
- simple_memcpy(_buff, buff, length);
- _position = length;
- }
- } else {
- flush();
- _stream->write(buff, length);
- }
-}
-
-void BufferedOutputStream::flush() {
- if (_position > 0) {
- _stream->write(_buff, _position);
- _position = 0;
- }
-}
-
-///////////////////////////////////////////////////////////
-
int32_t InputBuffer::read(void * buff, uint32_t length) {
uint32_t rd = _capacity - _position < length ? _capacity - _position : length;
if (rd > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
index 541bab5..e945b95 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h
@@ -20,49 +20,12 @@
#define BUFFERSTREAM_H_
#include <string>
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
using std::string;
-class BufferedInputStream : public FilterInputStream {
-protected:
- char * _buff;
- uint32_t _position;
- uint32_t _limit;
- uint32_t _capacity;
-public:
- BufferedInputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024);
-
- virtual ~BufferedInputStream();
-
- virtual void seek(uint64_t position);
-
- virtual uint64_t tell();
-
- virtual int32_t read(void * buff, uint32_t length);
-};
-
-class BufferedOutputStream : public FilterOutputStream {
-protected:
- char * _buff;
- uint32_t _position;
- uint32_t _capacity;
-
-public:
- BufferedOutputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024);
-
- virtual ~BufferedOutputStream();
-
- virtual uint64_t tell();
-
- virtual void write(const void * buff, uint32_t length);
-
- virtual void flush();
-
-};
-
class InputBuffer : public InputStream {
protected:
const char * _buff;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
index d07f2fb..d40abb9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc
@@ -16,84 +16,15 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <string>
+
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "util/WritableUtils.h"
-#include "Buffers.h"
+#include "lib/Buffers.h"
namespace NativeTask {
-DynamicBuffer::DynamicBuffer()
- : _data(NULL), _capacity(0), _size(0), _used(0) {
-}
-
-DynamicBuffer::DynamicBuffer(uint32_t capacity)
- : _data(NULL), _capacity(0), _size(0), _used(0) {
- reserve(capacity);
-}
-
-DynamicBuffer::~DynamicBuffer() {
- release();
-}
-
-void DynamicBuffer::release() {
- if (_data != NULL) {
- free(_data);
- _data = NULL;
- _capacity = 0;
- _used = 0;
- }
-}
-
-void DynamicBuffer::reserve(uint32_t capacity) {
- if (_data != NULL) {
- if (capacity > _capacity) {
- char * newdata = (char*)realloc(_data, capacity);
- if (newdata == NULL) {
- THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve realloc %u failed",
- capacity);
- }
- _data = newdata;
- _capacity = capacity;
- }
- return;
- }
- release();
- char * newdata = (char*)malloc(capacity);
- if (newdata == NULL) {
- THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve new %u failed", capacity);
- }
- _data = newdata;
- _capacity = capacity;
- _size = 0;
- _used = 0;
-}
-
-int32_t DynamicBuffer::refill(InputStream * stream) {
- if (_data == NULL || freeSpace() == 0) {
- THROW_EXCEPTION(IOException, "refill DynamicBuffer failed, no space left");
- }
- int32_t rd = stream->read(_data + _size, freeSpace());
- if (rd > 0) {
- _size += rd;
- }
- return rd;
-}
-
-void DynamicBuffer::cleanUsed() {
- if (_used > 0) {
- uint32_t needToMove = _size - _used;
- if (needToMove > 0) {
- memmove(_data, _data + _used, needToMove);
- _size = needToMove;
- } else {
- _size = 0;
- }
- _used = 0;
- }
-}
-
-///////////////////////////////////////////////////////////
ReadBuffer::ReadBuffer()
: _buff(NULL), _remain(0), _size(0), _capacity(0), _stream(NULL), _source(NULL) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
index 0da23af..13cd545 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
@@ -19,69 +19,12 @@
#ifndef BUFFERS_H_
#define BUFFERS_H_
-#include "Streams.h"
-#include "Compressions.h"
-#include "Constants.h"
+#include "lib/Streams.h"
+#include "lib/Compressions.h"
+#include "lib/Constants.h"
namespace NativeTask {
-class DynamicBuffer {
-protected:
- char * _data;
- uint32_t _capacity;
- uint32_t _size;
- uint32_t _used;
-public:
- DynamicBuffer();
-
- DynamicBuffer(uint32_t capacity);
-
- ~DynamicBuffer();
-
- void reserve(uint32_t capacity);
-
- void release();
-
- uint32_t capacity() {
- return _capacity;
- }
-
- char * data() {
- return _data;
- }
-
- uint32_t size() {
- return _size;
- }
-
- uint32_t used() {
- return _used;
- }
-
- char * current() {
- return _data + _used;
- }
-
- char * end() {
- return _data + _size;
- }
-
- uint32_t remain() {
- return _size - _used;
- }
-
- uint32_t freeSpace() {
- return _capacity - _size;
- }
-
- void use(uint32_t count) {
- _used += count;
- }
-
- void cleanUsed();
-
- int32_t refill(InputStream * stream);
-};
/**
* A lightweight read buffer, act as buffered input stream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
deleted file mode 100644
index b426fc3..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "Combiner.h"
-#include "StringUtil.h"
-
-namespace NativeTask {
-
-NativeCombineRunner::NativeCombineRunner(Config * config, ObjectCreatorFunc combinerCreator)
- : _config(config), _combinerCreator(combinerCreator), _keyGroupCount(0) {
- if (NULL == _combinerCreator) {
- THROW_EXCEPTION_EX(UnsupportException, "Create combiner failed");
- }
-}
-
-KeyGroupIterator * NativeCombineRunner::createKeyGroupIterator(KVIterator * iter) {
- return new KeyGroupIteratorImpl(iter);
-}
-
-void NativeCombineRunner::combine(CombineContext context, KVIterator * iterator,
- IFileWriter * writer) {
- Configurable * combiner = (Configurable *)(_combinerCreator());
- if (NULL != combiner) {
- combiner->configure(_config);
- }
-
- NativeObjectType type = combiner->type();
- switch (type) {
- case MapperType: {
- Mapper * mapper = (Mapper*)combiner;
- mapper->setCollector(writer);
-
- Buffer key;
- Buffer value;
- while (iterator->next(key, value)) {
- mapper->map(key.data(), key.length(), value.data(), value.length());
- }
- mapper->close();
- delete mapper;
- }
- break;
- case ReducerType: {
- Reducer * reducer = (Reducer*)combiner;
- reducer->setCollector(writer);
- KeyGroupIterator * kg = createKeyGroupIterator(iterator);
- while (kg->nextKey()) {
- _keyGroupCount++;
- reducer->reduce(*kg);
- }
- reducer->close();
- delete reducer;
- }
- break;
- default:
- THROW_EXCEPTION(UnsupportException, "Combiner type not support");
- }
-}
-
-} /* namespace NativeTask */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
index 09f4eb0..a7309ff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h
@@ -18,7 +18,7 @@
#ifndef COMBINER_H_
#define COMBINER_H_
#include "commons.h"
-#include "IFile.h"
+#include "lib/IFile.h"
namespace NativeTask {
@@ -66,21 +66,5 @@ public:
}
};
-class NativeCombineRunner : public ICombineRunner {
-private:
- Config * _config;
- ObjectCreatorFunc _combinerCreator;
- uint32_t _keyGroupCount;
-
-public:
- NativeCombineRunner(Config * config, ObjectCreatorFunc objectCreator);
-
-public:
- void combine(CombineContext type, KVIterator * kvIterator, IFileWriter * writer);
-
-private:
- KeyGroupIterator * createKeyGroupIterator(KVIterator * iter);
-};
-
} /* namespace NativeTask */
#endif /* COMBINER_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
index 0f954e0..879c016 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "config.h"
-#include "SyncUtils.h"
-#include "Compressions.h"
+#include "lib/Compressions.h"
+#include "util/SyncUtils.h"
#include "codec/GzipCodec.h"
#include "codec/SnappyCodec.h"
#include "codec/Lz4Codec.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
index f70adb6..96c5084 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h
@@ -21,7 +21,7 @@
#include <string>
#include <vector>
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
index e137883..e0a698c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc
@@ -20,14 +20,14 @@
#include <fcntl.h>
#include <dirent.h>
#include <sys/stat.h>
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
#include "NativeTask.h"
-#include "TaskCounters.h"
-#include "NativeObjectFactory.h"
-#include "Path.h"
-#include "FileSystem.h"
+#include "lib/TaskCounters.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/Path.h"
+#include "lib/FileSystem.h"
namespace NativeTask {
@@ -122,14 +122,15 @@ void FileOutputStream::close() {
/////////////////////////////////////////////////////////////
class RawFileSystem : public FileSystem {
-protected:
+ protected:
string getRealPath(const string & path) {
if (StringUtil::StartsWith(path, "file:")) {
return path.substr(5);
}
return path;
}
-public:
+
+ public:
InputStream * open(const string & path) {
return new FileInputStream(getRealPath(path));
}
@@ -206,7 +207,7 @@ public:
struct stat sb;
if (stat(np.c_str(), &sb) == 0) {
- if (S_ISDIR (sb.st_mode) == 0) {
+ if (S_ISDIR(sb.st_mode) == 0) {
return 1;
}
return 0;
@@ -226,7 +227,7 @@ public:
if (mkdir(npath, nmode)) {
return 1;
}
- } else if (S_ISDIR (sb.st_mode) == 0) {
+ } else if (S_ISDIR(sb.st_mode) == 0) {
return 1;
}
*p++ = '/'; /* restore slash */
@@ -259,4 +260,4 @@ FileSystem & FileSystem::getLocal() {
return RawFileSystemInstance;
}
-} // namespace Hadoap
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
index 7649947..8eaf32d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h
@@ -21,7 +21,7 @@
#include <string>
#include "NativeTask.h"
-#include "Streams.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
index 4a5edda..2d3e0b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
-#include "IFile.h"
-#include "Compressions.h"
+#include "lib/IFile.h"
+#include "lib/Compressions.h"
#include "lib/FileSystem.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
index d86e98a..e397f90 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h
@@ -19,11 +19,11 @@
#ifndef IFILE_H_
#define IFILE_H_
-#include "Checksum.h"
-#include "Buffers.h"
-#include "WritableUtils.h"
-#include "SpillInfo.h"
-#include "MapOutputSpec.h"
+#include "util/Checksum.h"
+#include "lib/Buffers.h"
+#include "util/WritableUtils.h"
+#include "lib/SpillInfo.h"
+#include "lib/MapOutputSpec.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
index dfde456..3d0516a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc
@@ -15,8 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#include "Iterator.h"
-#include "commons.h"
+#include "lib/Iterator.h"
+#include "lib/commons.h"
namespace NativeTask {
@@ -88,4 +88,4 @@ bool KeyGroupIteratorImpl::next() {
return result;
}
-} //namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
index 2eaa342..1e36e18 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "Log.h"
+#include "lib/Log.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
index d151f1f..1175b23 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
@@ -16,19 +16,21 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <string>
+
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "FileSystem.h"
-#include "NativeObjectFactory.h"
-#include "MapOutputCollector.h"
-#include "Merge.h"
+#include "lib/FileSystem.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/MapOutputCollector.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
@@ -36,22 +38,18 @@ ICombineRunner * CombineRunnerWrapper::createCombiner() {
ICombineRunner * combineRunner = NULL;
if (NULL != _config->get(NATIVE_COMBINER)) {
- const char * combinerClass = _config->get(NATIVE_COMBINER);
- ObjectCreatorFunc objectCreater = NativeObjectFactory::GetObjectCreator(combinerClass);
- if (NULL == objectCreater) {
- THROW_EXCEPTION_EX(UnsupportException, "Combiner not found: %s", combinerClass);
- } else {
- LOG("[MapOutputCollector::configure] native combiner is enabled: %s", combinerClass);
- }
- combineRunner = new NativeCombineRunner(_config, objectCreater);
+ // Earlier versions of this code supported user-defined
+ // native Combiner implementations. This simplified version
+ // no longer supports it.
+ THROW_EXCEPTION_EX(UnsupportException, "Native Combiners not supported");
+ }
+
+ CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler();
+ if (NULL != javaCombiner) {
+ _isJavaCombiner = true;
+ combineRunner = (ICombineRunner *)javaCombiner;
} else {
- CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler();
- if (NULL != javaCombiner) {
- _isJavaCombiner = true;
- combineRunner = (ICombineRunner *)javaCombiner;
- } else {
- LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java");
- }
+ LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java");
}
return combineRunner;
}
@@ -118,7 +116,7 @@ void MapOutputCollector::init(uint32_t defaultBlockSize, uint32_t memoryCapacity
_pool->init(memoryCapacity);
- //TODO: add support for customized comparator
+ // TODO: add support for customized comparator
this->_keyComparator = keyComparator;
_buckets = new PartitionBucket*[_numPartitions];
@@ -160,7 +158,8 @@ void MapOutputCollector::configure(Config * config) {
uint32_t capacity = config->getInt(MAPRED_IO_SORT_MB, 300) * 1024 * 1024;
uint32_t defaultBlockSize = getDefaultBlockSize(capacity, _numPartitions, maxBlockSize);
- LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024,
+ LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, "
+ "max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024,
maxBlockSize / 1024, capacity / 1024 / 1024);
ComparatorPtr comparator = getComparator(config, _spec);
@@ -179,7 +178,8 @@ void MapOutputCollector::configure(Config * config) {
KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t kvlength) {
PartitionBucket * partition = getPartition(partitionId);
if (NULL == partition) {
- THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d", partitionId, _numPartitions);
+ THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d",
+ partitionId, _numPartitions);
}
KVBuffer * dest = partition->allocateKVBuffer(kvlength);
@@ -301,7 +301,7 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
info->path = spillOutput;
uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime;
- const uint64_t M = 1000000; //million
+ const uint64_t M = 1000000; // million
LOG("%s-spill: { id: %d, collect: %"PRIu64" ms, "
"in-memory sort: %"PRIu64" ms, in-memory records: %"PRIu64", "
"merge&spill: %"PRIu64" ms, uncompressed size: %"PRIu64", "
@@ -369,7 +369,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
uint64_t recordCount;
writer->getStatistics(outputSize, realOutputSize, recordCount);
- const uint64_t M = 1000000; //million
+ const uint64_t M = 1000000; // million
LOG("Final-merge-spill: { id: %d, in-memory sort: %"PRIu64" ms, "
"in-memory records: %"PRIu64", merge&spill: %"PRIu64" ms, "
"records: %"PRIu64", uncompressed size: %"PRIu64", "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
index 474b8b1..ee447ed 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
@@ -20,14 +20,14 @@
#define MAP_OUTPUT_COLLECTOR_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "PartitionBucket.h"
+#include "lib/MemoryPool.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
+#include "lib/PartitionBucket.h"
#include "lib/SpillOutputService.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
index 5088206..8874d02 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc
@@ -16,8 +16,8 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "MapOutputSpec.h"
+#include "lib/commons.h"
+#include "lib/MapOutputSpec.h"
#include "NativeTask.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
index fac96ab..ce43997 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h
@@ -20,8 +20,8 @@
#define MAPOUTPUTSPEC_H_
#include <string>
-#include "Checksum.h"
-#include "WritableUtils.h"
+#include "util/Checksum.h"
+#include "util/WritableUtils.h"
#include "NativeTask.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
index 29f59b9..b3734a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc
@@ -15,18 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+#include <algorithm>
+
#include "NativeTask.h"
-#include "commons.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "MapOutputSpec.h"
+#include "lib/commons.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
-#include "MemoryBlock.h"
-#include "MemoryPool.h"
+#include "lib/MemoryBlock.h"
+#include "lib/MemoryPool.h"
#include "util/DualPivotQuickSort.h"
namespace NativeTask {
@@ -62,4 +64,4 @@ void MemoryBlock::sort(SortAlgorithm type, ComparatorPtr comparator) {
}
_sorted = true;
}
-} //namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
index 0303fb2..1aa9cc2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h
@@ -19,8 +19,8 @@
#ifndef MEMORYPOOL_H_
#define MEMORYPOOL_H_
-#include "Buffers.h"
-#include "MapOutputSpec.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
#include "NativeTask.h"
#include "util/StringUtil.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
index 126b82d..0d18db8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "Merge.h"
+#include "lib/Merge.h"
#include "lib/FileSystem.h"
namespace NativeTask {
@@ -34,7 +34,6 @@ Merger::Merger(IFileWriter * writer, Config * config, ComparatorPtr comparator,
ICombineRunner * combineRunner)
: _writer(writer), _config(config), _combineRunner(combineRunner), _first(true),
_comparator(comparator) {
-
}
Merger::~Merger() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
index 956fa41..cf1f2be 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h
@@ -20,10 +20,10 @@
#define MERGE_H_
#include "NativeTask.h"
-#include "Buffers.h"
-#include "MapOutputCollector.h"
-#include "IFile.h"
-#include "MinHeap.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputCollector.h"
+#include "lib/IFile.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
index ab434f6..5885f34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h
@@ -20,7 +20,7 @@
#define MIN_HEAP_H_
#include "NativeTask.h"
-#include "Buffers.h"
+#include "lib/Buffers.h"
template<typename T, typename Compare>
void heapify(T* first, int rt, int heap_len, Compare & Comp) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
index 237b55e..5d69b26 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc
@@ -18,9 +18,9 @@
#include <dlfcn.h>
-#include "commons.h"
-#include "NativeObjectFactory.h"
-#include "NativeLibrary.h"
+#include "lib/commons.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/NativeLibrary.h"
namespace NativeTask {
@@ -30,7 +30,6 @@ namespace NativeTask {
NativeLibrary::NativeLibrary(const string & path, const string & name)
: _path(path), _name(name), _getObjectCreatorFunc(NULL), _functionGetter(NULL) {
-
}
bool NativeLibrary::init() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
index ce3799d..2185798 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
@@ -20,11 +20,11 @@
#ifndef __CYGWIN__
#include <execinfo.h>
#endif
-#include "commons.h"
+#include "lib/commons.h"
#include "NativeTask.h"
-#include "NativeObjectFactory.h"
-#include "NativeLibrary.h"
-#include "BufferStream.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/NativeLibrary.h"
+#include "lib/BufferStream.h"
#include "util/StringUtil.h"
#include "util/SyncUtils.h"
#include "util/WritableUtils.h"
@@ -53,19 +53,10 @@ extern "C" void handler(int sig) {
}
DEFINE_NATIVE_LIBRARY(NativeTask) {
- //signal(SIGSEGV, handler);
REGISTER_CLASS(BatchHandler, NativeTask);
REGISTER_CLASS(CombineHandler, NativeTask);
REGISTER_CLASS(MCollectorOutputHandler, NativeTask);
- REGISTER_CLASS(Mapper, NativeTask);
- REGISTER_CLASS(Reducer, NativeTask);
- REGISTER_CLASS(Partitioner, NativeTask);
- REGISTER_CLASS(Folder, NativeTask);
NativeObjectFactory::SetDefaultClass(BatchHandlerType, "NativeTask.BatchHandler");
- NativeObjectFactory::SetDefaultClass(MapperType, "NativeTask.Mapper");
- NativeObjectFactory::SetDefaultClass(ReducerType, "NativeTask.Reducer");
- NativeObjectFactory::SetDefaultClass(PartitionerType, "NativeTask.Partitioner");
- NativeObjectFactory::SetDefaultClass(FolderType, "NativeTask.Folder");
}
namespace NativeTask {
@@ -132,7 +123,7 @@ bool NativeObjectFactory::Init() {
}
const char * version = GetConfig().get(NATIVE_HADOOP_VERSION);
LOG("[NativeObjectLibrary] NativeTask library initialized with hadoop %s",
- version==NULL?"unkown":version);
+ version == NULL ? "unkown" : version);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
index 28aeaf3..be682e8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
@@ -20,9 +20,9 @@
#include "org_apache_hadoop_mapred_nativetask_NativeRuntime.h"
#endif
#include "config.h"
-#include "commons.h"
-#include "jniutils.h"
-#include "NativeObjectFactory.h"
+#include "lib/commons.h"
+#include "lib/jniutils.h"
+#include "lib/NativeObjectFactory.h"
using namespace NativeTask;
@@ -36,8 +36,7 @@ using namespace NativeTask;
* Signature: ([B)Z
*/
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
- (JNIEnv *jenv, jclass clazz, jbyteArray codec)
-{
+ (JNIEnv *jenv, jclass clazz, jbyteArray codec) {
const std::string codecString = JNU_ByteArrayToString(jenv, codec);
if ("org.apache.hadoop.io.compress.GzipCodec" == codecString) {
return JNI_TRUE;
@@ -58,7 +57,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
JNIEnv * jenv, jclass nativeRuntimeClass) {
try {
NativeTask::NativeObjectFactory::Release();
@@ -82,7 +82,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIConfigure
* Signature: ([[B)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure(
JNIEnv * jenv, jclass nativeRuntimeClass, jobjectArray configs) {
try {
NativeTask::Config & config = NativeTask::NativeObjectFactory::GetConfig();
@@ -139,7 +140,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNICreateDefaultNativeObject
* Signature: ([B)J
- */JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject(
+ */
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject(
JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray type) {
try {
std::string typeString = JNU_ByteArrayToString(jenv, type);
@@ -166,7 +168,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIReleaseNativeObject
* Signature: (J)V
- */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject(
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject(
JNIEnv * jenv, jclass nativeRuntimeClass, jlong objectAddr) {
try {
NativeTask::NativeObject * nobj = ((NativeTask::NativeObject *)objectAddr);
@@ -196,7 +199,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRegisterModule
* Signature: ([B[B)I
- */JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule(
+ */
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule(
JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray modulePath, jbyteArray moduleName) {
try {
std::string pathString = JNU_ByteArrayToString(jenv, modulePath);
@@ -225,7 +229,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIUpdateStatus
* Signature: ()[B
- */JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus(
+ */
+JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus(
JNIEnv * jenv, jclass nativeRuntimeClass) {
try {
std::string statusData;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
index 5c42e56..5dc880c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc
@@ -18,11 +18,10 @@
#ifndef __CYGWIN__
#include <execinfo.h>
#endif
-#include "commons.h"
-#include "util/Hash.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "NativeTask.h"
-#include "NativeObjectFactory.h"
+#include "lib/NativeObjectFactory.h"
namespace NativeTask {
@@ -34,20 +33,6 @@ const string NativeObjectTypeToString(NativeObjectType type) {
switch (type) {
case BatchHandlerType:
return string("BatchHandlerType");
- case MapperType:
- return string("MapperType");
- case ReducerType:
- return string("ReducerType");
- case PartitionerType:
- return string("PartitionerType");
- case CombinerType:
- return string("CombinerType");
- case FolderType:
- return string("FolderType");
- case RecordReaderType:
- return string("RecordReaderType");
- case RecordWriterType:
- return string("RecordWriterType");
default:
return string("UnknownObjectType");
}
@@ -56,20 +41,6 @@ const string NativeObjectTypeToString(NativeObjectType type) {
NativeObjectType NativeObjectTypeFromString(const string type) {
if (type == "BatchHandlerType") {
return BatchHandlerType;
- } else if (type == "MapperType") {
- return MapperType;
- } else if (type == "ReducerType") {
- return ReducerType;
- } else if (type == "PartitionerType") {
- return PartitionerType;
- } else if (type == "CombinerType") {
- return CombinerType;
- } else if (type == "FolderType") {
- return CombinerType;
- } else if (type == "RecordReaderType") {
- return RecordReaderType;
- } else if (type == "RecordWriterType") {
- return RecordWriterType;
}
return UnknownObjectType;
}
@@ -96,7 +67,7 @@ HadoopException::HadoopException(const string & what) {
#ifndef __CYGWIN__
size = backtrace(array, 64);
char ** traces = backtrace_symbols(array, size);
- for (size_t i = 0; i< size;i++) {
+ for (size_t i = 0; i < size; i++) {
_reason.append("\n\t");
_reason.append(traces[i]);
}
@@ -235,13 +206,6 @@ Counter * ProcessorBase::getCounter(const string & group, const string & name) {
return NULL;
}
-uint32_t Partitioner::getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition) {
- if (numPartition == 1) {
- return 0;
- }
- return (Hash::BytesHash(key, keyLen) & 0x7fffffff) % numPartition;
-}
-
///////////////////////////////////////////////////////////
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
index 55ed4b2..33c2403 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc
@@ -16,19 +16,19 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "NativeObjectFactory.h"
-#include "PartitionBucket.h"
-#include "Merge.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/PartitionBucket.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
-#include "PartitionBucketIterator.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
+#include "lib/PartitionBucketIterator.h"
namespace NativeTask {
@@ -39,7 +39,8 @@ KVIterator * PartitionBucket::getIterator() {
return new PartitionBucketIterator(this, _keyComparator);
}
-void PartitionBucket::spill(IFileWriter * writer) throw (IOException, UnsupportException) {
+void PartitionBucket::spill(IFileWriter * writer)
+ throw(IOException, UnsupportException) {
KVIterator * iterator = getIterator();
if (NULL == iterator || NULL == writer) {
return;
@@ -71,6 +72,4 @@ void PartitionBucket::sort(SortAlgorithm type) {
_sorted = true;
}
-}
-;
-// namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
index 3be52ff..cd9ec97 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
@@ -20,14 +20,14 @@
#define PARTITION_BUCKET_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "MemoryBlock.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
+#include "lib/MemoryPool.h"
+#include "lib/MemoryBlock.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
index 0b26f21..ade042e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc
@@ -16,18 +16,20 @@
* limitations under the License.
*/
-#include "commons.h"
+#include <algorithm>
+
+#include "lib/commons.h"
#include "util/Timer.h"
#include "util/StringUtil.h"
-#include "NativeObjectFactory.h"
-#include "PartitionBucketIterator.h"
-#include "Merge.h"
+#include "lib/NativeObjectFactory.h"
+#include "lib/PartitionBucketIterator.h"
+#include "lib/Merge.h"
#include "NativeTask.h"
-#include "WritableUtils.h"
+#include "util/WritableUtils.h"
#include "util/DualPivotQuickSort.h"
-#include "Combiner.h"
-#include "TaskCounters.h"
-#include "MinHeap.h"
+#include "lib/Combiner.h"
+#include "lib/TaskCounters.h"
+#include "lib/MinHeap.h"
namespace NativeTask {
@@ -108,7 +110,5 @@ bool PartitionBucketIterator::next(Buffer & key, Buffer & value) {
}
return false;
}
-}
-;
-// namespace NativeTask
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
index beabff8..be78497 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h
@@ -20,14 +20,14 @@
#define PARTITION_BUCKET_ITERATOR_H_
#include "NativeTask.h"
-#include "MemoryPool.h"
-#include "Timer.h"
-#include "Buffers.h"
-#include "MapOutputSpec.h"
-#include "IFile.h"
-#include "SpillInfo.h"
-#include "Combiner.h"
-#include "PartitionBucket.h"
+#include "lib/MemoryPool.h"
+#include "util/Timer.h"
+#include "lib/Buffers.h"
+#include "lib/MapOutputSpec.h"
+#include "lib/IFile.h"
+#include "lib/SpillInfo.h"
+#include "lib/Combiner.h"
+#include "lib/PartitionBucket.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
index 6d85fe2..78321e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "Path.h"
+#include "lib/Path.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
index bce2cb2..9cff529 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "Streams.h"
-#include "FileSystem.h"
-#include "Buffers.h"
-#include "SpillInfo.h"
+#include "lib/commons.h"
+#include "lib/Streams.h"
+#include "lib/FileSystem.h"
+#include "lib/Buffers.h"
+#include "lib/SpillInfo.h"
namespace NativeTask {
@@ -69,5 +69,5 @@ void SingleSpillInfo::writeSpillInfo(const std::string & filepath) {
delete fout;
}
-}
+} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
index c2d64c9..9e8ffba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-#include "commons.h"
-#include "Checksum.h"
-#include "Streams.h"
+#include "lib/commons.h"
+#include "util/Checksum.h"
+#include "lib/Streams.h"
namespace NativeTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
index 39a7c43..b70db4c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "TaskCounters.h"
+#include "lib/TaskCounters.h"
namespace NativeTask {
@@ -36,6 +36,5 @@ const char * TaskCounters::FILESYSTEM_COUNTER_GROUP = "FileSystemCounters";
DEFINE_COUNTER(FILE_BYTES_READ)
DEFINE_COUNTER(FILE_BYTES_WRITTEN)
-;
} // namespace NativeTask
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
deleted file mode 100644
index 3f96faf..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef TRACKING_COLLECTOR_H
-#define TRACKING_COLLECTOR_H
-
-#include <stdint.h>
-#include <string>
-
-namespace NativeTask {
-
-class TrackingCollector : public Collector {
-protected:
- Collector * _collector;
- Counter * _counter;
-public:
- TrackingCollector(Collector * collector, Counter * counter)
- : _collector(collector), _counter(counter) {
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) {
- _counter->increase();
- _collector->collect(key, keyLen, value, valueLen);
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen,
- int32_t partition) {
- _counter->increase();
- _collector->collect(key, keyLen, value, valueLen, partition);
- }
-};
-
-} //namespace NativeTask
-
-#endif //TRACKING_COLLECTOR_H
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
index 4212533..57500b7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
@@ -41,13 +41,12 @@
#include <map>
#include <algorithm>
-#include "primitives.h"
-#include "Log.h"
+#include "lib/primitives.h"
+#include "lib/Log.h"
#include "NativeTask.h"
-#include "Constants.h"
+#include "lib/Constants.h"
-#include "Iterator.h"
-#include "TrackingCollector.h"
+#include "lib/Iterator.h"
#endif /* COMMONS_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
index 01eaa57..d1c45bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-#include "commons.h"
+#include "lib/commons.h"
#include "util/StringUtil.h"
#include "util/SyncUtils.h"
-#include "jniutils.h"
+#include "lib/jniutils.h"
using namespace NativeTask;
@@ -53,7 +53,7 @@ JavaVM * JNU_GetJVM(void) {
JavaVMOption options[noArgs];
options[0].optionString = optHadoopClassPath;
- //Create the VM
+ // Create the VM
JavaVMInitArgs vm_args;
vm_args.version = JNI_VERSION_1_6;
vm_args.options = options;
@@ -108,4 +108,4 @@ std::string JNU_ByteArrayToString(JNIEnv * jenv, jbyteArray src) {
return ret;
}
return std::string();
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
index 4c4f3cc..191e093 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
@@ -17,7 +17,7 @@
*/
#include <assert.h>
-#include "Checksum.h"
+#include "util/Checksum.h"
namespace NativeTask {