You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/13 05:54:29 UTC
[23/50] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc
warnings. Contributed by Manu Zhang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 11712ab..60bb6f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -622,7 +622,6 @@ typedef void (*ANY_FUNC_PTR)();
#define DEFINE_NATIVE_LIBRARY(Library) \
static std::map<std::string, NativeTask::ObjectCreatorFunc> Library##ClassMap__; \
extern "C" void * Library##GetFunctionGetter(const std::string & name) { \
- void * ret = NULL; \
std::map<std::string, NativeTask::ObjectCreatorFunc>::iterator itr = Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return (void *)(itr->second); \
@@ -630,7 +629,6 @@ typedef void (*ANY_FUNC_PTR)();
return NULL; \
} \
extern "C" NativeTask::ObjectCreatorFunc Library##GetObjectCreator(const std::string & name) { \
- NativeObject * ret = NULL; \
std::map<std::string, NativeTask::ObjectCreatorFunc>::iterator itr = Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return itr->second; \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
index a59b9d8..59cd8fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
@@ -23,7 +23,7 @@
namespace NativeTask {
BlockCompressStream::BlockCompressStream(OutputStream * stream, uint32_t bufferSizeHint)
- : CompressStream(stream), _compressedBytesWritten(0), _tempBufferSize(0), _tempBuffer(NULL) {
+ : CompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0), _compressedBytesWritten(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
}
@@ -68,7 +68,7 @@ uint64_t BlockCompressStream::compressedBytesWritten() {
//////////////////////////////////////////////////////////////
BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t bufferSizeHint)
- : DecompressStream(stream), _tempBufferSize(0), _tempBuffer(NULL) {
+ : DecompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
_tempDecompressBuffer = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
index bc022a8..15ce2ea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
@@ -103,10 +103,10 @@ protected:
*/
inline void output(const char * buff, uint32_t length) {
while (length > 0) {
- if (length > _out.remain()) {
+ uint32_t remain = _out.remain();
+ if (length > remain) {
flushOutput();
}
- uint32_t remain = _out.remain();
uint32_t cp = length < remain ? length : remain;
simple_memcpy(_out.current(), buff, cp);
buff += cp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
index 40a32df..83da238 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
@@ -24,9 +24,9 @@ const int LENGTH_OF_REFILL_STRING = 6;
const Command CombineHandler::COMBINE(4, "Combine");
CombineHandler::CombineHandler()
- : _combineContext(NULL), _kvIterator(NULL), _writer(NULL), _config(NULL), _kvCached(false),
- _kType(UnknownType), _vType(UnknownType), _combineInputRecordCount(0), _combineInputBytes(0),
- _combineOutputRecordCount(0),_combineOutputBytes(0){
+ : _combineContext(NULL), _kvIterator(NULL), _writer(NULL), _kType(UnknownType),
+ _vType(UnknownType), _config(NULL), _kvCached(false), _combineInputRecordCount(0),
+ _combineInputBytes(0), _combineOutputRecordCount(0), _combineOutputBytes(0) {
}
CombineHandler::~CombineHandler() {
@@ -95,7 +95,6 @@ uint32_t CombineHandler::feedDataToJavaInWritableSerialization() {
* KV: key or value
*/
void CombineHandler::outputKeyOrValue(SerializeInfo & KV, KeyValueType type) {
- uint32_t length = 0;
switch (type) {
case TextType:
output(KV.varBytes, KV.outerLength - KV.buffer.length());
@@ -159,7 +158,6 @@ uint32_t CombineHandler::feedDataToJava(SerializationFramework serializationType
void CombineHandler::handleInput(ByteBuffer & in) {
char * buff = in.current();
uint32_t length = in.remain();
- const char * end = buff + length;
uint32_t remain = length;
char * pos = buff;
if (_asideBuffer.remain() > 0) {
@@ -183,7 +181,7 @@ void CombineHandler::handleInput(ByteBuffer & in) {
THROW_EXCEPTION(IOException, "k/v meta information incomplete");
}
- int kvLength = kvBuffer->lengthConvertEndium();
+ uint32_t kvLength = kvBuffer->lengthConvertEndium();
if (kvLength > remain) {
_asideBytes.resize(kvLength);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
index cc6b5e9..0da23af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h
@@ -429,8 +429,8 @@ public:
void rewind(int newPos, int newLimit) {
this->_position = newPos;
- if (newLimit > this->_capacity) {
- THROW_EXCEPTION(IOException, "length larger than input buffer capacity");
+ if (newLimit < 0 || newLimit > this->_capacity) {
+ THROW_EXCEPTION(IOException, "length smaller than zero or larger than input buffer capacity");
}
this->_limit = newLimit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
index ff272ba..d11d823 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc
@@ -27,9 +27,9 @@ namespace NativeTask {
///////////////////////////////////////////////////////////
IFileReader::IFileReader(InputStream * stream, SingleSpillInfo * spill, bool deleteInputStream)
- : _deleteSourceStream(deleteInputStream), _stream(stream), _source(NULL),
- _checksumType(spill->checkSumType), _kType(spill->keyType), _vType(spill->valueType),
- _codec(spill->codec), _segmentIndex(-1), _spillInfo(spill), _valuePos(NULL), _valueLen(0) {
+ : _stream(stream), _source(NULL), _checksumType(spill->checkSumType), _kType(spill->keyType),
+ _vType(spill->valueType), _codec(spill->codec), _segmentIndex(-1), _spillInfo(spill),
+ _valuePos(NULL), _valueLen(0), _deleteSourceStream(deleteInputStream) {
_source = new ChecksumInputStream(_stream, _checksumType);
_source->setLimit(0);
_reader.init(128 * 1024, _source, _codec);
@@ -97,9 +97,8 @@ IFileWriter * IFileWriter::create(const std::string & filepath, const MapOutputS
IFileWriter::IFileWriter(OutputStream * stream, ChecksumType checksumType, KeyValueType ktype,
KeyValueType vtype, const string & codec, Counter * counter, bool deleteTargetStream)
- : _deleteTargetStream(deleteTargetStream), _stream(stream), _dest(NULL),
- _checksumType(checksumType), _kType(ktype), _vType(vtype), _codec(codec),
- _recordCounter(counter) {
+ : _stream(stream), _dest(NULL), _checksumType(checksumType), _kType(ktype), _vType(vtype),
+ _codec(codec), _recordCounter(counter), _deleteTargetStream(deleteTargetStream) {
_dest = new ChecksumOutputStream(_stream, _checksumType);
_appendBuffer.init(128 * 1024, _dest, _codec);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
index 34f7072..d2b116d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc
@@ -76,9 +76,9 @@ void CombineRunnerWrapper::combine(CombineContext type, KVIterator * kvIterator,
/////////////////////////////////////////////////////////////////
MapOutputCollector::MapOutputCollector(uint32_t numberPartitions, SpillOutputService * spillService)
- : _config(NULL), _buckets(NULL), _keyComparator(NULL), _defaultBlockSize(0),
- _combineRunner(NULL), _spilledRecords(NULL), _spillOutput(spillService), _pool(NULL),
- _numPartitions(numberPartitions) {
+ : _config(NULL), _numPartitions(numberPartitions), _buckets(NULL), _keyComparator(NULL),
+ _combineRunner(NULL), _spilledRecords(NULL), _spillOutput(spillService),
+ _defaultBlockSize(0), _pool(NULL) {
_pool = new MemoryPool();
}
@@ -186,7 +186,7 @@ KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t k
delete spillpath;
}
- dest = dest = partition->allocateKVBuffer(kvlength);
+ dest = partition->allocateKVBuffer(kvlength);
if (NULL == dest) {
// io.sort.mb too small, cann't proceed
// should not get here, cause get_buffer_to_put can throw OOM exception
@@ -294,7 +294,7 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput,
uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime;
LOG(
- "[MapOutputCollector::mid_spill] Sort and spill: {spilled file path: %s, id: %d, collect: %llu ms, sort: %llu ms, spill: %llu ms, records: %llu, uncompressed total bytes: %llu, compressed total bytes: %llu}",
+ "[MapOutputCollector::mid_spill] Sort and spill: {spilled file path: %s, id: %d, collect: %"PRIu64" ms, sort: %"PRIu64" ms, spill: %"PRIu64" ms, records: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64"}",
info->path.c_str(), _spillInfos.getSpillCount(), collecttime / M, metrics.sortTime / M, spillTime / M,
metrics.recordCount, info->getEndPosition(), info->getRealEndPosition());
@@ -339,7 +339,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
SortMetrics metrics;
sortPartitions(_spec.sortOrder, _spec.sortAlgorithm, NULL, metrics);
- LOG("[MapOutputCollector::mid_spill] Sort final in memory kvs: {sort: %llu ms, records: %llu}",
+ LOG("[MapOutputCollector::mid_spill] Sort final in memory kvs: {sort: %"PRIu64" ms, records: %"PRIu64"}",
metrics.sortTime / M, metrics.recordCount);
merger->addMergeEntry(new MemoryMergeEntry(_buckets, _numPartitions));
@@ -347,7 +347,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath,
Timer timer;
merger->merge();
LOG(
- "[MapOutputCollector::final_merge_and_spill] Merge and Spill:{spilled file id: %d, merge and spill time: %llu ms}",
+ "[MapOutputCollector::final_merge_and_spill] Merge and Spill:{spilled file id: %d, merge and spill time: %"PRIu64" ms}",
_spillInfos.getSpillCount(), (timer.now() - timer.last()) / M);
delete merger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
index 0692a33..b7150ac 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h
@@ -55,8 +55,8 @@ private:
public:
CombineRunnerWrapper(Config * config, SpillOutputService * service)
- : _spillOutput(service), _config(config), _isJavaCombiner(false), _combineRunner(NULL),
- _combinerInited(false) {
+ : _config(config), _combineRunner(NULL), _isJavaCombiner(false),
+ _combinerInited(false), _spillOutput(service) {
}
~CombineRunnerWrapper() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
index a8d7e1b..5434980 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc
@@ -162,7 +162,7 @@ void Merger::merge() {
_writer->getStatistics(output_size, real_output_size);
if (total_record != 0) {
- LOG("[Merge] Merged segment#: %lu, record#: %llu, avg record size: %llu, uncompressed total bytes: %llu, compressed total bytes: %llu, time: %llu ms",
+ LOG("[Merge] Merged segment#: %lu, record#: %"PRIu64", avg record size: %"PRIu64", uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
_entries.size(),
total_record,
output_size / (total_record),
@@ -170,7 +170,7 @@ void Merger::merge() {
real_output_size,
interval / M);
} else {
- LOG("[Merge] Merged segments#, %lu, uncompressed total bytes: %llu, compressed total bytes: %llu, time: %llu ms",
+ LOG("[Merge] Merged segments#, %lu, uncompressed total bytes: %"PRIu64", compressed total bytes: %"PRIu64", time: %"PRIu64" ms",
_entries.size(),
output_size,
real_output_size,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
index 37c45af..cf5e33b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h
@@ -50,8 +50,8 @@ private:
public:
PartitionBucket(MemoryPool * pool, uint32_t partition, ComparatorPtr comparator,
ICombineRunner * combineRunner, uint32_t blockSize)
- : _pool(pool), _partition(partition), _keyComparator(comparator),
- _combineRunner(combineRunner), _blockSize(blockSize), _sorted(false) {
+ : _pool(pool), _partition(partition), _blockSize(blockSize),
+ _keyComparator(comparator), _combineRunner(combineRunner), _sorted(false) {
if (NULL == _pool || NULL == comparator) {
THROW_EXCEPTION_EX(IOException, "pool is NULL, or comparator is not set");
}
@@ -102,11 +102,11 @@ public:
}
_sorted = false;
MemoryBlock * memBlock = NULL;
- uint32_t memBockSize = _memBlocks.size();
- if (memBockSize > 0) {
- memBlock = _memBlocks[memBockSize - 1];
+ uint32_t memBlockSize = _memBlocks.size();
+ if (memBlockSize > 0) {
+ memBlock = _memBlocks[memBlockSize - 1];
}
- if (NULL != memBockSize && memBlock->remainSpace() >= kvLength) {
+ if (NULL != memBlock && memBlock->remainSpace() >= kvLength) {
return memBlock->allocateKVBuffer(kvLength);
} else {
uint32_t min = kvLength;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
index 89770ad..4212533 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h
@@ -30,6 +30,9 @@
#include <memory.h>
#include <fcntl.h>
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
#include <limits>
#include <string>
#include <vector>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
index 4e5b94a..7baaa0c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc
@@ -36,19 +36,19 @@ string StringUtil::ToString(uint32_t v) {
string StringUtil::ToString(int64_t v) {
char tmp[32];
- snprintf(tmp, 32, "%lld", (long long int)v);
+ snprintf(tmp, 32, "%"PRId64, v);
return tmp;
}
string StringUtil::ToString(int64_t v, char pad, int64_t len) {
char tmp[32];
- snprintf(tmp, 32, "%%%c%lldlld", pad, len);
+ snprintf(tmp, 32, "%%%c%"PRId64""PRId64, pad, len);
return Format(tmp, v);
}
string StringUtil::ToString(uint64_t v) {
char tmp[32];
- snprintf(tmp, 32, "%llu", (long long unsigned int)v);
+ snprintf(tmp, 32, "%"PRIu64, v);
return tmp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
index de02ac7..1b99286 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
@@ -181,13 +181,11 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl
char * dest = new char[blockSize + 8];
CompressResult result;
Timer t;
- int compressedSize;
for (size_t start = 0; start < data.length(); start += blockSize) {
size_t currentblocksize = std::min(data.length() - start, blockSize);
uint64_t startTime = t.now();
for (int i = 0; i < times; i++) {
int osize = LZ4_compress((char*)data.data() + start, outputBuffer, currentblocksize);
- compressedSize = osize;
result.compressedSize += osize;
result.uncompressedSize += currentblocksize;
}
@@ -197,6 +195,7 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl
for (int i = 0; i < times; i++) {
// memset(dest, 0, currentblocksize+8);
int osize = LZ4_uncompress(outputBuffer, dest, currentblocksize);
+ ASSERT_EQ(currentblocksize, osize);
// printf("%016llx blocksize: %lu\n", bswap64(*(uint64_t*)(dest+currentblocksize)), currentblocksize);
}
endTime = t.now();
@@ -239,7 +238,7 @@ void MeasureSingleFileSnappy(const string & path, CompressResult & total, size_t
char * dest = new char[blockSize];
CompressResult result;
Timer t;
- int compressedSize;
+ int compressedSize = -1;
for (size_t start = 0; start < data.length(); start += blockSize) {
size_t currentblocksize = std::min(data.length() - start, blockSize);
uint64_t startTime = t.now();
@@ -272,7 +271,7 @@ TEST(Perf, RawCompressionSnappy) {
vector<FileEntry> inputfiles;
FileSystem::getLocal().list(inputdir, inputfiles);
CompressResult total;
- printf("Block size: %lldK\n", blockSize / 1024);
+ printf("Block size: %"PRId64"K\n", blockSize / 1024);
for (size_t i = 0; i < inputfiles.size(); i++) {
if (!inputfiles[i].isDirectory) {
MeasureSingleFileSnappy((inputdir + "/" + inputfiles[i].name).c_str(), total, blockSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
index 1d8b520..955ef29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
@@ -119,13 +119,17 @@ void TestIFileWriteRead2(vector<pair<string, string> > & kvs, char * buff, size_
InputBuffer inputBuffer = InputBuffer(buff, outputBuffer.tell());
IFileReader * ir = new IFileReader(&inputBuffer, info);
timer.reset();
+ int sum = 0;
while (ir->nextPartition()) {
const char * key, *value;
uint32_t keyLen, valueLen;
while (NULL != (key = ir->nextKey(keyLen))) {
value = ir->value(valueLen);
+ sum += value[0];
}
}
+ // use the result so that value() calls don't get optimized out
+ ASSERT_NE(0xdeadbeef, sum);
LOG("%s",
timer.getSpeedM2(" Read data", info->getEndPosition(), info->getRealEndPosition()).c_str());
delete ir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
index 228cace..c03f02e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc
@@ -35,7 +35,7 @@ TEST(ByteBuffer, read) {
ASSERT_EQ(buff, byteBuffer.current());
ASSERT_EQ(0, byteBuffer.remain());
- int newPos = byteBuffer.advance(3);
+ byteBuffer.advance(3);
ASSERT_EQ(3, byteBuffer.current() - byteBuffer.base());
byteBuffer.rewind(10, 20);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
index cd8fb27..a2dd909 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc
@@ -43,7 +43,7 @@ TEST(FixSizeContainer, test) {
container->fill(toBeFilled.c_str(), toBeFilled.length());
- for (int i = 0; i < container->position(); i++) {
+ for (uint32_t i = 0; i < container->position(); i++) {
char * c = container->base() + i;
ASSERT_EQ(toBeFilled[i], *c);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
index 6e79796..88a3258 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc
@@ -88,8 +88,6 @@ void TestKeyGroupIterator() {
int * keyPtr = (int *)key;
const char * value = NULL;
while (NULL != (value = groupIterator->nextValue(length))) {
- int * valuePtr = (int *)value;
-
if (actualKeyCount.find(*keyPtr) == actualKeyCount.end()) {
actualKeyCount[*keyPtr] = 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
index 87fba15..c11d5ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc
@@ -27,13 +27,12 @@ namespace NativeTask {
TEST(MemoryBlockIterator, test) {
const uint32_t BUFFER_LENGTH = 100;
- const uint32_t BLOCK_ID = 3;
char * bytes = new char[BUFFER_LENGTH];
MemoryBlock block(bytes, BUFFER_LENGTH);
const uint32_t KV_SIZE = 60;
- KVBuffer * kv1 = block.allocateKVBuffer(KV_SIZE);
- KVBuffer * kv2 = block.allocateKVBuffer(KV_SIZE);
+ block.allocateKVBuffer(KV_SIZE);
+ block.allocateKVBuffer(KV_SIZE);
MemBlockIterator iter(&block);
@@ -50,7 +49,6 @@ class MemoryBlockFactory {
public:
static MemoryBlock * create(std::vector<int> & keys) {
const uint32_t BUFFER_LENGTH = 1000;
- const uint32_t BLOCK_ID = 3;
char * bytes = new char[BUFFER_LENGTH];
MemoryBlock * block1 = new MemoryBlock(bytes, BUFFER_LENGTH);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
index dc60ac3..1d48c79 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc
@@ -44,6 +44,7 @@ TEST(MemoryBlock, test) {
ASSERT_EQ(2, block.getKVCount());
ASSERT_EQ(kv1, block.getKVBuffer(0));
+ ASSERT_EQ(kv2, block.getKVBuffer(1));
ASSERT_EQ(BUFFER_LENGTH - 2 * KV_SIZE, block.remainSpace());
ASSERT_EQ(false, block.sorted());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
index e2f854c..501e005 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc
@@ -24,7 +24,7 @@ static uint64_t test_length(int64_t len, size_t size, size_t loopTime) {
TestConfig.setInt(GenerateLen, len);
Generate(data, size, "bytes");
Timer t;
- uint64_t ret;
+ uint64_t ret = 0;
for (size_t m = 0; m < loopTime; m++) {
for (size_t i = 0; i < data.size(); i++) {
ret += Hash::BytesHash(data[i].c_str(), data[i].length());