You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/12/12 07:01:50 UTC
[1/2] kudu git commit: KUDU-100 enable the ability to use RLE for the
int64 type
Repository: kudu
Updated Branches:
refs/heads/master 2eba7b884 -> 244c6feae
KUDU-100 enable the ability to use RLE for the int64 type
As suggested in https://gerrit.cloudera.org/#/c/4822/, I modified code in cfile with a test.
Change-Id: I36ca437fcf0c98b3d79f7c07d72c1a7e61ff6a2f
Reviewed-on: http://gerrit.cloudera.org:8080/4946
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2e403f23
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2e403f23
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2e403f23
Branch: refs/heads/master
Commit: 2e403f232af835e51107c20883678306e169f90d
Parents: 2eba7b8
Author: honghaijei <ho...@gmail.com>
Authored: Fri Nov 4 12:25:15 2016 +0000
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Dec 12 06:55:26 2016 +0000
----------------------------------------------------------------------
src/kudu/cfile/cfile-test-base.h | 29 +++++++++++++++++++++++++++++
src/kudu/cfile/cfile-test.cc | 12 ++++++++++++
src/kudu/cfile/encoding-test.cc | 5 ++++-
src/kudu/cfile/type_encodings.cc | 6 ++----
src/kudu/util/bit-util.h | 4 ++--
5 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e403f23/src/kudu/cfile/cfile-test-base.h
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile-test-base.h b/src/kudu/cfile/cfile-test-base.h
index aa1ca8c..2828838 100644
--- a/src/kudu/cfile/cfile-test-base.h
+++ b/src/kudu/cfile/cfile-test-base.h
@@ -213,6 +213,25 @@ class Int32DataGenerator : public DataGenerator<INT32, HAS_NULLS> {
}
};
+template<bool HAS_NULLS>
+class UInt64DataGenerator : public DataGenerator<UINT64, HAS_NULLS> {
+ public:
+ UInt64DataGenerator() {}
+ uint64_t BuildTestValue(size_t /*block_index*/, size_t value) OVERRIDE {
+ return value * 0x123456789abcdefULL;
+ }
+};
+
+template<bool HAS_NULLS>
+class Int64DataGenerator : public DataGenerator<INT64, HAS_NULLS> {
+ public:
+ Int64DataGenerator() {}
+ int64_t BuildTestValue(size_t /*block_index*/, size_t value) OVERRIDE {
+ int64_t r = (value * 0x123456789abcdefULL) & 0x7fffffffffffffffULL;
+ return value % 2 == 0 ? r : -r;
+ }
+};
+
// Floating-point data generator.
// This works for both floats and doubles.
template<DataType DATA_TYPE, bool HAS_NULLS>
@@ -475,6 +494,16 @@ void TimeReadFile(FsManager* fs_manager, const BlockId& block_id, size_t *count_
TimeReadFileForDataType<INT32, int64_t>(iter, count);
break;
}
+ case UINT64:
+ {
+ TimeReadFileForDataType<UINT64, uint64_t>(iter, count);
+ break;
+ }
+ case INT64:
+ {
+ TimeReadFileForDataType<INT64, uint64_t>(iter, count);
+ break;
+ }
case FLOAT:
{
TimeReadFileForDataType<FLOAT, float>(iter, count);
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e403f23/src/kudu/cfile/cfile-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile-test.cc b/src/kudu/cfile/cfile-test.cc
index 1db6b76..67cd888 100644
--- a/src/kudu/cfile/cfile-test.cc
+++ b/src/kudu/cfile/cfile-test.cc
@@ -454,6 +454,18 @@ TEST_P(TestCFileBothCacheTypes, TestReadWriteInt32) {
}
}
+TEST_P(TestCFileBothCacheTypes, TestReadWriteUInt64) {
+ for (auto enc : { PLAIN_ENCODING, RLE, BIT_SHUFFLE }) {
+ TestReadWriteFixedSizeTypes<UInt64DataGenerator<false>>(enc);
+ }
+}
+
+TEST_P(TestCFileBothCacheTypes, TestReadWriteInt64) {
+ for (auto enc : { PLAIN_ENCODING, RLE, BIT_SHUFFLE }) {
+ TestReadWriteFixedSizeTypes<Int64DataGenerator<false>>(enc);
+ }
+}
+
TEST_P(TestCFileBothCacheTypes, TestFixedSizeReadWritePlainEncodingFloat) {
TestReadWriteFixedSizeTypes<FPDataGenerator<FLOAT, false> >(PLAIN_ENCODING);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e403f23/src/kudu/cfile/encoding-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/encoding-test.cc b/src/kudu/cfile/encoding-test.cc
index f667bf7..0a2e368 100644
--- a/src/kudu/cfile/encoding-test.cc
+++ b/src/kudu/cfile/encoding-test.cc
@@ -36,7 +36,9 @@
#include "kudu/util/group_varint-inl.h"
#include "kudu/util/hexdump.h"
#include "kudu/util/memory/arena.h"
+#include "kudu/util/random.h"
#include "kudu/util/test_macros.h"
+#include <kudu/util/test_util.h>
#include "kudu/util/stopwatch.h"
using std::unique_ptr;
@@ -463,8 +465,9 @@ class TestEncoding : public ::testing::Test {
srand(123);
std::vector<CppType> to_insert;
+ Random rd(SeedRandom());
for (int i = 0; i < 10003; i++) {
- auto val = random() % std::numeric_limits<CppType>::max();
+ int64_t val = rd.Next64() % std::numeric_limits<CppType>::max();
// For signed types, randomly use both negative and positive values.
if (std::numeric_limits<CppType>::is_signed && (random() % 2) == 1) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e403f23/src/kudu/cfile/type_encodings.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/type_encodings.cc b/src/kudu/cfile/type_encodings.cc
index f789c03..36250ac 100644
--- a/src/kudu/cfile/type_encodings.cc
+++ b/src/kudu/cfile/type_encodings.cc
@@ -235,10 +235,6 @@ class TypeEncodingResolver {
// Add the encoding mappings
// the first encoder/decoder to be
// added to the mapping becomes the default
- //
- // TODO: Fix/work around the issue with RLE/BitWriter which
- // (currently) makes it impossible to use RLE with
- // 64-bit int types.
private:
TypeEncodingResolver() {
AddMapping<UINT8, PLAIN_ENCODING>();
@@ -260,8 +256,10 @@ class TypeEncodingResolver {
AddMapping<INT32, RLE>();
AddMapping<INT32, BIT_SHUFFLE>();
AddMapping<UINT64, PLAIN_ENCODING>();
+ AddMapping<UINT64, RLE>();
AddMapping<UINT64, BIT_SHUFFLE>();
AddMapping<INT64, PLAIN_ENCODING>();
+ AddMapping<INT64, RLE>();
AddMapping<INT64, BIT_SHUFFLE>();
AddMapping<FLOAT, PLAIN_ENCODING>();
AddMapping<FLOAT, BIT_SHUFFLE>();
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e403f23/src/kudu/util/bit-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/bit-util.h b/src/kudu/util/bit-util.h
index 21f1558..5f36887 100644
--- a/src/kudu/util/bit-util.h
+++ b/src/kudu/util/bit-util.h
@@ -40,12 +40,12 @@ class BitUtil {
}
static inline uint64_t ShiftLeftZeroOnOverflow(uint64_t v, int num_bits) {
- if (num_bits >= 64) return 0;
+ if (PREDICT_FALSE(num_bits >= 64)) return 0;
return v << num_bits;
}
static inline uint64_t ShiftRightZeroOnOverflow(uint64_t v, int num_bits) {
- if (num_bits >= 64) return 0;
+ if (PREDICT_FALSE(num_bits >= 64)) return 0;
return v >> num_bits;
}
[2/2] kudu git commit: Modify the default value of log_dir flag.
Posted by to...@apache.org.
Modify the default value of log_dir flag.
The document about configuration says that the default value of
log_dir flag is "/var/log/kudu" but it's "/tmp" in fact.
Change-Id: Icdc6539907e6ba105bf08b0ee349dbeb4089e08d
Reviewed-on: http://gerrit.cloudera.org:8080/5436
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/244c6fea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/244c6fea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/244c6fea
Branch: refs/heads/master
Commit: 244c6feaed418a4d38eca3d328db4d77d7552471
Parents: 2e403f2
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Authored: Fri Dec 9 11:58:40 2016 +0900
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Dec 12 06:57:40 2016 +0000
----------------------------------------------------------------------
docs/configuration.adoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/244c6fea/docs/configuration.adoc
----------------------------------------------------------------------
diff --git a/docs/configuration.adoc b/docs/configuration.adoc
index 22b591f..53a66ec 100644
--- a/docs/configuration.adoc
+++ b/docs/configuration.adoc
@@ -76,7 +76,7 @@ directories where the Master will place its data blocks.
|--fs_wal_dir | string | | The directory where the Master will
place its write-ahead logs. May be the same as _one of_ the directories listed in
`--fs_data_dirs`, but not a sub-directory of a data directory.
-|--log_dir | string | /var/log/kudu | The directory to store Master log files.
+|--log_dir | string | /tmp | The directory to store Master log files.
|===
For the full list of flags for masters, see the
@@ -98,7 +98,7 @@ of directories where the Tablet Server will place its data blocks.
|--fs_wal_dir | string | | The directory where the Tablet Server will
place its write-ahead logs. May be the same as _one of_ the directories listed in
`--fs_data_dirs`, but not a sub-directory of a data directory.
-|--log_dir | string | /var/log/kudu | The directory to store Tablet Server log files
+|--log_dir | string | /tmp | The directory to store Tablet Server log files
|--tserver_master_addrs | string | `127.0.0.1:7051` | Comma separated
addresses of the masters which the tablet server should connect to. The masters
do not read this flag.