You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/02/05 18:27:58 UTC
[1/2] kudu git commit: [tests] Add more test coverage for INT128
columns
Repository: kudu
Updated Branches:
refs/heads/master bf6c2c07b -> c19a42170
[tests] Add more test coverage for INT128 columns
Improves test coverage of reading and writing INT128
columns and fixes a predicate issue exposed by those
tests.
Also adds INT64 tests where it was missing.
Change-Id: I3350634e55aad4316036c80d03e95e0acdf64295
Reviewed-on: http://gerrit.cloudera.org:8080/9193
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2c89bd71
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2c89bd71
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2c89bd71
Branch: refs/heads/master
Commit: 2c89bd71cfaa4005095cfb44d46a222c48614a34
Parents: bf6c2c0
Author: Grant Henke <gr...@gmail.com>
Authored: Fri Feb 2 11:27:07 2018 -0600
Committer: Grant Henke <gr...@gmail.com>
Committed: Mon Feb 5 18:14:50 2018 +0000
----------------------------------------------------------------------
src/kudu/cfile/cfile-test.cc | 15 +++++++--
src/kudu/cfile/encoding-test.cc | 35 +++++++++++++++++---
src/kudu/client/scan_batch.cc | 6 ++++
src/kudu/common/column_predicate.cc | 2 ++
src/kudu/common/partial_row.cc | 10 ++++++
.../tablet/all_types-scan-correctness-test.cc | 6 ++++
6 files changed, 66 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/cfile/cfile-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile-test.cc b/src/kudu/cfile/cfile-test.cc
index db129a5..afbdd17 100644
--- a/src/kudu/cfile/cfile-test.cc
+++ b/src/kudu/cfile/cfile-test.cc
@@ -64,6 +64,7 @@
#include "kudu/util/cache.h"
#include "kudu/util/compression/compression.pb.h"
#include "kudu/util/env.h"
+#include "kudu/util/int128.h"
#include "kudu/util/mem_tracker.h"
#include "kudu/util/memory/arena.h"
#include "kudu/util/metrics.h"
@@ -548,14 +549,19 @@ TEST_P(TestCFileBothCacheTypes, TestReadWriteInt64) {
}
}
+TEST_P(TestCFileBothCacheTypes, TestReadWriteInt128) {
+ TestReadWriteFixedSizeTypes<Int128DataGenerator<false>>(PLAIN_ENCODING);
+}
+
TEST_P(TestCFileBothCacheTypes, TestFixedSizeReadWritePlainEncodingFloat) {
- TestReadWriteFixedSizeTypes<FPDataGenerator<FLOAT, false> >(PLAIN_ENCODING);
+ TestReadWriteFixedSizeTypes<FPDataGenerator<FLOAT, false>>(PLAIN_ENCODING);
}
TEST_P(TestCFileBothCacheTypes, TestFixedSizeReadWritePlainEncodingDouble) {
- TestReadWriteFixedSizeTypes<FPDataGenerator<DOUBLE, false> >(PLAIN_ENCODING);
+ TestReadWriteFixedSizeTypes<FPDataGenerator<DOUBLE, false>>(PLAIN_ENCODING);
}
-// Test for BitShuffle builder for UINT8, INT8, UINT16, INT16, UINT32, INT32, FLOAT, DOUBLE
+// Test for BitShuffle builder for UINT8, INT8, UINT16, INT16, UINT32, INT32,
+// UINT64, INT64, INT128, FLOAT, DOUBLE
template <typename T>
class BitShuffleTest : public TestCFile {
public:
@@ -569,6 +575,9 @@ typedef ::testing::Types<UInt8DataGenerator<false>,
Int16DataGenerator<false>,
UInt32DataGenerator<false>,
Int32DataGenerator<false>,
+ UInt64DataGenerator<false>,
+ Int64DataGenerator<false>,
+ Int128DataGenerator<false>,
FPDataGenerator<FLOAT, false>,
FPDataGenerator<DOUBLE, false> > MyTypes;
TYPED_TEST_CASE(BitShuffleTest, MyTypes);
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/cfile/encoding-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/encoding-test.cc b/src/kudu/cfile/encoding-test.cc
index d8691ea..e13b6c7 100644
--- a/src/kudu/cfile/encoding-test.cc
+++ b/src/kudu/cfile/encoding-test.cc
@@ -48,6 +48,7 @@
#include "kudu/gutil/port.h"
#include "kudu/gutil/stringprintf.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/int128.h"
#include "kudu/util/group_varint-inl.h"
#include "kudu/util/hexdump.h"
#include "kudu/util/memory/arena.h"
@@ -690,8 +691,8 @@ TEST_F(TestEncoding, TestPlainBlockEncoder) {
PlainBlockDecoder<INT32> >(ints.get(), kSize);
}
-// Test for bitshuffle block, for INT32, FLOAT, DOUBLE
-TEST_F(TestEncoding, TestBShufIntBlockEncoder) {
+// Test for bitshuffle block, for INT32, INT64, INT128, FLOAT, DOUBLE
+TEST_F(TestEncoding, TestBShufInt32BlockEncoder) {
const uint32_t kSize = 10000;
gscoped_ptr<int32_t[]> ints(new int32_t[kSize]);
@@ -703,6 +704,30 @@ TEST_F(TestEncoding, TestBShufIntBlockEncoder) {
BShufBlockDecoder<INT32> >(ints.get(), kSize);
}
+TEST_F(TestEncoding, TestBShufInt64BlockEncoder) {
+ const uint32_t kSize = 10000;
+
+ gscoped_ptr<int64_t[]> ints(new int64_t[kSize]);
+ for (int i = 0; i < kSize; i++) {
+ ints.get()[i] = random();
+ }
+
+ TestEncodeDecodeTemplateBlockEncoder<INT64, BShufBlockBuilder<INT64>,
+ BShufBlockDecoder<INT64> >(ints.get(), kSize);
+}
+
+TEST_F(TestEncoding, TestBShufInt128BlockEncoder) {
+ const uint32_t kSize = 10000;
+
+ gscoped_ptr<int128_t[]> ints(new int128_t[kSize]);
+ for (int i = 0; i < kSize; i++) {
+ ints.get()[i] = random();
+ }
+
+ TestEncodeDecodeTemplateBlockEncoder<INT128, BShufBlockBuilder<INT128>,
+ BShufBlockDecoder<INT128> >(ints.get(), kSize);
+}
+
TEST_F(TestEncoding, TestBShufFloatBlockEncoder) {
const uint32_t kSize = 10000;
@@ -877,7 +902,7 @@ TYPED_TEST(IntEncodingTest, TestSeekAllTypes) {
this->template DoIntSeekTest<INT32>(10000, 1000, true);
this->template DoIntSeekTest<UINT64>(10000, 1000, true);
this->template DoIntSeekTest<INT64>(10000, 1000, true);
- // TODO: Uncomment when adding 128 bit support to RLE
+ // TODO: Uncomment when adding 128 bit support to RLE (KUDU-2284)
// this->template DoIntSeekTest<INT128>();
}
@@ -890,7 +915,7 @@ TYPED_TEST(IntEncodingTest, IntSeekTestTinyBlockAllTypes) {
this->template DoIntSeekTestTinyBlock<INT32>();
this->template DoIntSeekTestTinyBlock<UINT64>();
this->template DoIntSeekTestTinyBlock<INT64>();
- // TODO: Uncomment when adding 128 bit support to RLE
+ // TODO: Uncomment when adding 128 bit support to RLE (KUDU-2284)
// this->template DoIntSeekTestTinyBlock<INT128>();
}
@@ -903,7 +928,7 @@ TYPED_TEST(IntEncodingTest, TestRoundTrip) {
this->template DoIntRoundTripTest<INT32>();
this->template DoIntRoundTripTest<UINT64>();
this->template DoIntRoundTripTest<INT64>();
- // TODO: Uncomment when adding 128 bit support to RLE
+ // TODO: Uncomment when adding 128 bit support to RLE (KUDU-2284)
// this->template DoIntRoundTripTest<INT128>();
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/client/scan_batch.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scan_batch.cc b/src/kudu/client/scan_batch.cc
index c424986..410d125 100644
--- a/src/kudu/client/scan_batch.cc
+++ b/src/kudu/client/scan_batch.cc
@@ -260,6 +260,9 @@ template
Status KuduScanBatch::RowPtr::Get<TypeTraits<INT64> >(const Slice& col_name, int64_t* val) const;
template
+Status KuduScanBatch::RowPtr::Get<TypeTraits<INT128> >(const Slice& col_name, int128_t* val) const;
+
+template
Status KuduScanBatch::RowPtr::Get<TypeTraits<UNIXTIME_MICROS> >(
const Slice& col_name, int64_t* val) const;
@@ -291,6 +294,9 @@ template
Status KuduScanBatch::RowPtr::Get<TypeTraits<INT64> >(int col_idx, int64_t* val) const;
template
+Status KuduScanBatch::RowPtr::Get<TypeTraits<INT128> >(int col_idx, int128_t* val) const;
+
+template
Status KuduScanBatch::RowPtr::Get<TypeTraits<UNIXTIME_MICROS> >(int col_idx, int64_t* val) const;
template
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/common/column_predicate.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate.cc b/src/kudu/common/column_predicate.cc
index d2bb583..1c784e6 100644
--- a/src/kudu/common/column_predicate.cc
+++ b/src/kudu/common/column_predicate.cc
@@ -599,6 +599,7 @@ bool ColumnPredicate::EvaluateCell(DataType type, const void* cell) const {
case INT16: return EvaluateCell<INT16>(cell);
case INT32: return EvaluateCell<INT32>(cell);
case INT64: return EvaluateCell<INT64>(cell);
+ case INT128: return EvaluateCell<INT128>(cell);
case UINT8: return EvaluateCell<UINT8>(cell);
case UINT16: return EvaluateCell<UINT16>(cell);
case UINT32: return EvaluateCell<UINT32>(cell);
@@ -618,6 +619,7 @@ void ColumnPredicate::Evaluate(const ColumnBlock& block, SelectionVector* sel) c
case INT16: return EvaluateForPhysicalType<INT16>(block, sel);
case INT32: return EvaluateForPhysicalType<INT32>(block, sel);
case INT64: return EvaluateForPhysicalType<INT64>(block, sel);
+ case INT128: return EvaluateForPhysicalType<INT128>(block, sel);
case UINT8: return EvaluateForPhysicalType<UINT8>(block, sel);
case UINT16: return EvaluateForPhysicalType<UINT16>(block, sel);
case UINT32: return EvaluateForPhysicalType<UINT32>(block, sel);
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/common/partial_row.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/partial_row.cc b/src/kudu/common/partial_row.cc
index de9dd6b..9ca3bcb 100644
--- a/src/kudu/common/partial_row.cc
+++ b/src/kudu/common/partial_row.cc
@@ -461,6 +461,11 @@ Status KuduPartialRow::Set<TypeTraits<INT64> >(int col_idx,
bool owned);
template
+Status KuduPartialRow::Set<TypeTraits<INT128> >(int col_idx,
+ const TypeTraits<INT128>::cpp_type& val,
+ bool owned);
+
+template
Status KuduPartialRow::Set<TypeTraits<UNIXTIME_MICROS> >(
int col_idx,
const TypeTraits<UNIXTIME_MICROS>::cpp_type& val,
@@ -512,6 +517,11 @@ Status KuduPartialRow::Set<TypeTraits<INT64> >(const Slice& col_name,
bool owned);
template
+Status KuduPartialRow::Set<TypeTraits<INT128> >(const Slice& col_name,
+ const TypeTraits<INT128>::cpp_type& val,
+ bool owned);
+
+template
Status KuduPartialRow::Set<TypeTraits<UNIXTIME_MICROS> >(
const Slice& col_name,
const TypeTraits<UNIXTIME_MICROS>::cpp_type& val,
http://git-wip-us.apache.org/repos/asf/kudu/blob/2c89bd71/src/kudu/tablet/all_types-scan-correctness-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/all_types-scan-correctness-test.cc b/src/kudu/tablet/all_types-scan-correctness-test.cc
index edf3a2c..cdf9e3d 100644
--- a/src/kudu/tablet/all_types-scan-correctness-test.cc
+++ b/src/kudu/tablet/all_types-scan-correctness-test.cc
@@ -578,6 +578,12 @@ typedef ::testing::Types<NumTypeRowOps<KeyTypeWrapper<INT8, BIT_SHUFFLE>>,
NumTypeRowOps<KeyTypeWrapper<INT32, RLE>>,
NumTypeRowOps<KeyTypeWrapper<INT64, BIT_SHUFFLE>>,
NumTypeRowOps<KeyTypeWrapper<INT64, PLAIN_ENCODING>>,
+ NumTypeRowOps<KeyTypeWrapper<INT64, BIT_SHUFFLE>>,
+ NumTypeRowOps<KeyTypeWrapper<INT64, RLE>>,
+ NumTypeRowOps<KeyTypeWrapper<INT128, BIT_SHUFFLE>>,
+ NumTypeRowOps<KeyTypeWrapper<INT128, PLAIN_ENCODING>>,
+ // TODO: Uncomment when adding 128 bit support to RLE (KUDU-2284)
+ // NumTypeRowOps<KeyTypeWrapper<INT128, RLE>>,
NumTypeRowOps<KeyTypeWrapper<FLOAT, BIT_SHUFFLE>>,
NumTypeRowOps<KeyTypeWrapper<FLOAT, PLAIN_ENCODING>>,
NumTypeRowOps<KeyTypeWrapper<DOUBLE, BIT_SHUFFLE>>,
[2/2] kudu git commit: [docs] Added recommendation to compress PK for
backfill inserts
Posted by da...@apache.org.
[docs] Added recommendation to compress PK for backfill inserts
Change-Id: I698d954265b4171e4d1bb7e01e286d0d489f1ec7
Reviewed-on: http://gerrit.cloudera.org:8080/9185
Reviewed-by: Dan Burkert <da...@cloudera.com>
Tested-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c19a4217
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c19a4217
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c19a4217
Branch: refs/heads/master
Commit: c19a42170576699d6f346faa716ae6f2fa665a97
Parents: 2c89bd7
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Thu Feb 1 16:16:45 2018 -0800
Committer: Dan Burkert <da...@cloudera.com>
Committed: Mon Feb 5 18:27:33 2018 +0000
----------------------------------------------------------------------
docs/schema_design.adoc | 84 ++++++++++++++++++++++++++++++++++----------
1 file changed, 65 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/c19a4217/docs/schema_design.adoc
----------------------------------------------------------------------
diff --git a/docs/schema_design.adoc b/docs/schema_design.adoc
index c5407a6..7f0e218 100644
--- a/docs/schema_design.adoc
+++ b/docs/schema_design.adoc
@@ -155,34 +155,80 @@ recommended to apply additional compression on top of this encoding.
[[primary-keys]]
== Primary Key Design
-Every Kudu table must declare a primary key index comprised of one or more
-columns. Primary key columns must be non-nullable, and may not be a boolean or
-floating-point type. Once set during table creation, the set of columns in the
-primary key may not be altered. Like an RDBMS primary key, the Kudu primary key
-enforces a uniqueness constraint; attempting to insert a row with the same
-primary key values as an existing row will result in a duplicate key error.
-
-Unlike an RDBMS, Kudu does not provide an auto-incrementing column feature, so
-the application must always provide the full primary key during insert. Row
-delete and update operations must also specify the full primary key of the row
-to be changed; Kudu does not natively support range deletes or updates. The
-primary key values of a column may not be updated after the row is inserted;
-however, the row may be deleted and re-inserted with the updated value.
+Every Kudu table must declare a primary key comprised of one or more columns.
+Like an RDBMS primary key, the Kudu primary key enforces a uniqueness constraint.
+Attempting to insert a row with the same primary key values as an existing row
+will result in a duplicate key error.
+
+Primary key columns must be non-nullable, and may not be a boolean or floating-
+point type.
+
+Once set during table creation, the set of columns in the primary key may not
+be altered.
+
+Unlike an RDBMS, Kudu does not provide an auto-incrementing column feature,
+so the application must always provide the full primary key during insert.
+
+Row delete and update operations must also specify the full primary key of the
+row to be changed. Kudu does not natively support range deletes or updates.
+
+The primary key values of a column may not be updated after the row is inserted.
+However, the row may be deleted and re-inserted with the updated value.
+
[[indexing]]
=== Primary Key Index
-As with many traditional relational databases, Kudu's primary key is a clustered
-index. All rows within a tablet are kept in primary key sorted order. Kudu scans
-which specify equality or range constraints on the primary key will
-automatically skip rows which can not satisfy the predicate. This allows
-individual rows to be efficiently found by specifying equality constraints on
-the primary key columns.
+As with many traditional relational databases, Kudu’s primary key is in a
+clustered index. All rows within a tablet are sorted by its primary key.
+
+When scanning Kudu rows, use equality or range predicates on primary key
+columns to efficiently find the rows.
NOTE: Primary key indexing optimizations apply to scans on individual tablets.
See the <<partition-pruning>> section for details on how scans can use
predicates to skip entire tablets.
+[[Backfilling]]
+=== Considerations for Backfill Inserts
+
+This section discuss a primary key design consideration for timeseries use
+cases where the primary key is a timestamp, or the first column of the primary
+key is a timestamp.
+
+Each time a row is inserted into a Kudu table, Kudu looks up the primary key in
+the primary key index storage to check whether that primary key is already
+present in the table. If the primary key exists in the table, a "duplicate key"
+error is returned. In the typical case where data is being inserted at
+the current time as it arrives from the data source, only a small range of
+primary keys are "hot". So, each of these "check for presence" operations is
+very fast. It hits the cached primary key storage in memory and doesn't require
+going to disk.
+
+In the case when you load historical data, which is called "backfilling", from
+an offline data source, each row that is inserted is likely to hit a cold area
+of the primary key index which is not resident in memory and will cause one or
+more HDD disk seeks. For example, in a normal ingestion case where Kudu sustains
+a few million inserts per second, the "backfill" use case might sustain only
+a few thousand inserts per second.
+
+To alleviate the performance issue during backfilling, consider the following
+options:
+
+* Make the primary keys more compressible.
++
+For example, with the first column of a primary key being a random ID of 32-bytes,
+caching one billion primary keys would require at least 32 GB of RAM to stay in
+cache. If caching backfill primary keys from several days ago, you need to have
+several times 32 GB of memory. By changing the primary key to be more compressible,
+you increase the likelihood that the primary keys can fit in cache and thus
+reducing the amount of random disk I/Os.
++
+
+* Use SSDs for storage as random seeks are orders of magnitude faster than spinning disks.
+
+* Change the primary key structure such that the backfill writes hit a continuous range of primary keys.
+
[[partitioning]]
== Partitioning