You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/04/01 05:15:16 UTC
[1/3] incubator-kudu git commit: Add internal IS NOT NULL predicate
type
Repository: incubator-kudu
Updated Branches:
refs/heads/branch-0.8.x fa70abe85 -> 60c155cf0
Add internal IS NOT NULL predicate type
This commit adds an internal IS NOT NULL predicate type, and changes LESS_EQUAL
predicate simplification to result in the IS NOT NULL predicate type when the
conversion to a LESS predicate fails and the column is nullable. This fixes
scans with predicates such as 'WHERE my_nullable_int8_col <= 127`. A followup
commit will contain comprehensive predicate tests covering this case.
Change-Id: Ifcf29b1f274df2ef5c5ac7a7a17cc06dfd59e191
Reviewed-on: http://gerrit.cloudera.org:8080/2671
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
(cherry picked from commit 9b8b020210905b0cc4dd81e0ca905e098d0cad33)
Reviewed-on: http://gerrit.cloudera.org:8080/2681
Reviewed-by: Jean-Daniel Cryans
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/2b13fafb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/2b13fafb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/2b13fafb
Branch: refs/heads/branch-0.8.x
Commit: 2b13fafb9fbb160075b3c2760a08757891030a8b
Parents: fa70abe
Author: Dan Burkert <da...@cloudera.com>
Authored: Wed Mar 30 15:29:10 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Mar 31 17:14:02 2016 +0000
----------------------------------------------------------------------
src/kudu/client/scan_predicate.cc | 2 +-
src/kudu/client/scanner-internal.cc | 4 ++
src/kudu/common/column_predicate-test.cc | 61 +++++++++++++++++++++++++--
src/kudu/common/column_predicate.cc | 48 +++++++++++++++++++--
src/kudu/common/column_predicate.h | 6 +++
src/kudu/common/common.proto | 3 ++
src/kudu/tserver/tablet_service.cc | 4 ++
7 files changed, 120 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/client/scan_predicate.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scan_predicate.cc b/src/kudu/client/scan_predicate.cc
index f11b8d1..cb3758c 100644
--- a/src/kudu/client/scan_predicate.cc
+++ b/src/kudu/client/scan_predicate.cc
@@ -61,10 +61,10 @@ ComparisonPredicateData::ComparisonPredicateData(ColumnSchema col,
op_(op),
val_(val) {
}
+
ComparisonPredicateData::~ComparisonPredicateData() {
}
-
Status ComparisonPredicateData::AddToScanSpec(ScanSpec* spec, Arena* arena) {
void* val_void;
RETURN_NOT_OK(val_->data_->CheckTypeAndGetPointer(col_.name(),
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/client/scanner-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scanner-internal.cc b/src/kudu/client/scanner-internal.cc
index 568f381..3a3fc11 100644
--- a/src/kudu/client/scanner-internal.cc
+++ b/src/kudu/client/scanner-internal.cc
@@ -122,6 +122,10 @@ void ColumnPredicateIntoPB(const ColumnPredicate& predicate,
}
return;
};
+ case PredicateType::IsNotNull: {
+ pb->mutable_is_not_null();
+ return;
+ };
case PredicateType::None: LOG(FATAL) << "None predicate may not be converted to protobuf";
}
LOG(FATAL) << "unknown predicate type";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/common/column_predicate-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate-test.cc b/src/kudu/common/column_predicate-test.cc
index 274a774..9585a24 100644
--- a/src/kudu/common/column_predicate-test.cc
+++ b/src/kudu/common/column_predicate-test.cc
@@ -188,20 +188,58 @@ class TestColumnPredicate : public KuduTest {
ColumnPredicate::None(column),
ColumnPredicate::None(column),
PredicateType::None);
+
+ // IS NOT NULL
+
+ // IS NOT NULL AND
+ // IS NOT NULL
+ // =
+ // IS NOT NULL
+ TestMerge(ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::IsNotNull(column),
+ PredicateType::IsNotNull);
+
+ // IS NOT NULL AND
+ // None
+ // =
+ // None
+ TestMerge(ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::None(column),
+ ColumnPredicate::None(column),
+ PredicateType::None);
+
+ // IS NOT NULL AND
+ // |
+ // =
+ // |
+ TestMerge(ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::Equality(column, &values[0]),
+ ColumnPredicate::Equality(column, &values[0]),
+ PredicateType::Equality);
+
+ // IS NOT NULL AND
+ // [------)
+ // =
+ // [------)
+ TestMerge(ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::Range(column, &values[0], &values[2]),
+ ColumnPredicate::Range(column, &values[0], &values[2]),
+ PredicateType::Range);
}
};
TEST_F(TestColumnPredicate, TestMerge) {
- TestMergeCombinations(ColumnSchema("c", INT8),
+ TestMergeCombinations(ColumnSchema("c", INT8, true),
vector<int8_t> { 0, 1, 2, 3, 4, 5, 6 });
- TestMergeCombinations(ColumnSchema("c", INT32),
+ TestMergeCombinations(ColumnSchema("c", INT32, true),
vector<int32_t> { -100, -10, -1, 0, 1, 10, 100 });
- TestMergeCombinations(ColumnSchema("c", STRING),
+ TestMergeCombinations(ColumnSchema("c", STRING, true),
vector<Slice> { "a", "b", "c", "d", "e", "f", "g" });
- TestMergeCombinations(ColumnSchema("c", BINARY),
+ TestMergeCombinations(ColumnSchema("c", BINARY, true),
vector<Slice> { Slice("", 0),
Slice("\0", 1),
Slice("\0\0", 2),
@@ -265,6 +303,21 @@ TEST_F(TestColumnPredicate, TestInclusiveRange) {
ASSERT_EQ(boost::none, ColumnPredicate::InclusiveRange(column, nullptr, &max, &arena));
}
{
+ ColumnSchema column("c", INT32, true);
+ int32_t zero = 0;
+ int32_t two = 2;
+ int32_t three = 3;
+ int32_t max = INT32_MAX;
+
+ ASSERT_EQ(ColumnPredicate::Range(column, &zero, &three),
+ ColumnPredicate::InclusiveRange(column, &zero, &two, &arena));
+ ASSERT_EQ(ColumnPredicate::Range(column, &zero, nullptr),
+ ColumnPredicate::InclusiveRange(column, &zero, &max, &arena));
+
+ ASSERT_EQ(ColumnPredicate::IsNotNull(column),
+ ColumnPredicate::InclusiveRange(column, nullptr, &max, &arena));
+ }
+ {
ColumnSchema column("c", STRING);
Slice zero("", 0);
Slice two("\0\0", 2);
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/common/column_predicate.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate.cc b/src/kudu/common/column_predicate.cc
index d1c8c67..e7aef58 100644
--- a/src/kudu/common/column_predicate.cc
+++ b/src/kudu/common/column_predicate.cc
@@ -67,7 +67,14 @@ boost::optional<ColumnPredicate> ColumnPredicate::InclusiveRange(ColumnSchema co
memcpy(buf, upper, size);
if (!key_util::IncrementCell(column, buf, arena)) {
if (lower == nullptr) {
- return boost::none;
+ if (column.is_nullable()) {
+ // If incrementing the upper bound fails and the column is nullable,
+ // then return an IS NOT NULL predicate, so that null values will be
+ // filtered.
+ return ColumnPredicate::IsNotNull(move(column));
+ } else {
+ return boost::none;
+ }
} else {
upper = nullptr;
}
@@ -78,6 +85,11 @@ boost::optional<ColumnPredicate> ColumnPredicate::InclusiveRange(ColumnSchema co
return ColumnPredicate::Range(move(column), lower, upper);
}
+ColumnPredicate ColumnPredicate::IsNotNull(ColumnSchema column) {
+ CHECK(column.is_nullable());
+ return ColumnPredicate(PredicateType::IsNotNull, move(column), nullptr, nullptr);
+}
+
ColumnPredicate ColumnPredicate::None(ColumnSchema column) {
return ColumnPredicate(PredicateType::None, move(column), nullptr, nullptr);
}
@@ -90,8 +102,9 @@ void ColumnPredicate::SetToNone() {
void ColumnPredicate::Simplify() {
switch (predicate_type_) {
- case PredicateType::None: return;
- case PredicateType::Equality: return;
+ case PredicateType::None:
+ case PredicateType::Equality:
+ case PredicateType::IsNotNull: return;
case PredicateType::Range: {
if (lower_ != nullptr && upper_ != nullptr) {
if (column_.type_info()->Compare(lower_, upper_) >= 0) {
@@ -121,6 +134,18 @@ void ColumnPredicate::Merge(const ColumnPredicate& other) {
MergeIntoEquality(other);
return;
};
+ case PredicateType::IsNotNull: {
+ // NOT NULL is less selective than all other predicate types, so the
+ // intersection of NOT NULL with any other predicate is just the other
+ // predicate.
+ //
+ // Note: this will no longer be true when an IS NULL predicate type is
+ // added.
+ predicate_type_ = other.predicate_type_;
+ lower_ = other.lower_;
+ upper_ = other.upper_;
+ return;
+ };
}
LOG(FATAL) << "unknown predicate type";
}
@@ -163,6 +188,7 @@ void ColumnPredicate::MergeIntoRange(const ColumnPredicate& other) {
}
return;
};
+ case PredicateType::IsNotNull: return;
}
LOG(FATAL) << "unknown predicate type";
}
@@ -189,6 +215,7 @@ void ColumnPredicate::MergeIntoEquality(const ColumnPredicate& other) {
}
return;
};
+ case PredicateType::IsNotNull: return;
}
LOG(FATAL) << "unknown predicate type";
}
@@ -260,6 +287,17 @@ void ColumnPredicate::Evaluate(const ColumnBlock& block, SelectionVector *sel) c
});
return;
};
+ case PredicateType::IsNotNull: {
+ if (!block.is_nullable()) return;
+ // TODO: make this more efficient by using bitwise operations on the
+ // null and selection vectors.
+ for (size_t i = 0; i < block.nrows(); i++) {
+ if (sel->IsRowSelected(i) && block.is_null(i)) {
+ BitmapClear(sel->mutable_bitmap(), i);
+ }
+ }
+ return;
+ }
}
LOG(FATAL) << "unknown predicate type";
}
@@ -282,6 +320,9 @@ string ColumnPredicate::ToString() const {
case PredicateType::Equality: {
return strings::Substitute("`$0` = $1", column_.name(), column_.Stringify(lower_));
};
+ case PredicateType::IsNotNull: {
+ return strings::Substitute("`$0` IS NOT NULL", column_.name());
+ };
}
LOG(FATAL) << "unknown predicate type";
}
@@ -310,6 +351,7 @@ int SelectivityRank(const ColumnPredicate& predicate) {
case PredicateType::None: return 0;
case PredicateType::Equality: return 1;
case PredicateType::Range: return 2;
+ case PredicateType::IsNotNull: return 3;
}
LOG(FATAL) << "unknown predicate type";
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/common/column_predicate.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate.h b/src/kudu/common/column_predicate.h
index c90d841..e1461e1 100644
--- a/src/kudu/common/column_predicate.h
+++ b/src/kudu/common/column_predicate.h
@@ -41,6 +41,9 @@ enum class PredicateType {
// A predicate which evaluates to true if the column value falls within a
// range.
Range,
+
+ // A predicate which evaluates to true if the value is not null.
+ IsNotNull,
};
// A predicate which can be evaluated over a block of column values.
@@ -90,6 +93,9 @@ class ColumnPredicate {
const void* upper,
Arena* arena);
+ // Creates a new IS NOT NULL predicate for the column.
+ static ColumnPredicate IsNotNull(ColumnSchema column);
+
// Returns the type of this predicate.
PredicateType predicate_type() const {
return predicate_type_;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/common/common.proto
----------------------------------------------------------------------
diff --git a/src/kudu/common/common.proto b/src/kudu/common/common.proto
index a04f9b4..a3505bb 100644
--- a/src/kudu/common/common.proto
+++ b/src/kudu/common/common.proto
@@ -299,8 +299,11 @@ message ColumnPredicatePB {
optional bytes value = 1;
}
+ message IsNotNull {}
+
oneof predicate {
Range range = 2;
Equality equality = 3;
+ IsNotNull is_not_null = 4;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2b13fafb/src/kudu/tserver/tablet_service.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_service.cc b/src/kudu/tserver/tablet_service.cc
index 044b59f..1082e38 100644
--- a/src/kudu/tserver/tablet_service.cc
+++ b/src/kudu/tserver/tablet_service.cc
@@ -1310,6 +1310,10 @@ static Status SetupScanSpec(const NewScanRequestPB& scan_pb,
ret->AddPredicate(ColumnPredicate::Equality(col, value));
break;
};
+ case ColumnPredicatePB::kIsNotNull: {
+ ret->AddPredicate(ColumnPredicate::IsNotNull(col));
+ break;
+ };
default: return Status::InvalidArgument("Unknown predicate type for column", col.name());
}
}
[2/3] incubator-kudu git commit: Change row projector schema equality
check to use physical column type
Posted by ad...@apache.org.
Change row projector schema equality check to use physical column type
The row projector uses the column's physical type (e.g. for TIMESTAMP the
physical type is INT64) to build codegen cache keys, but used the normal column
type when comparing the projection schemas retrieved from the cache. This
resulted in spurious debug check failures.
Change-Id: I5e9a8d9c003be3942d3ce6f93d880adb45d8ded6
Reviewed-on: http://gerrit.cloudera.org:8080/2672
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
(cherry picked from commit 8f4726d83e6b6b7d76c59d06fde9231f0bb113b9)
Reviewed-on: http://gerrit.cloudera.org:8080/2682
Reviewed-by: Jean-Daniel Cryans
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/aa391149
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/aa391149
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/aa391149
Branch: refs/heads/branch-0.8.x
Commit: aa391149a4f39ff662316faffaca22c7b3783216
Parents: 2b13faf
Author: Dan Burkert <da...@cloudera.com>
Authored: Wed Mar 30 15:32:10 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Mar 31 17:14:07 2016 +0000
----------------------------------------------------------------------
src/kudu/codegen/row_projector.cc | 4 ++--
src/kudu/common/schema.h | 7 ++++++-
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/aa391149/src/kudu/codegen/row_projector.cc
----------------------------------------------------------------------
diff --git a/src/kudu/codegen/row_projector.cc b/src/kudu/codegen/row_projector.cc
index dd16285..d95d05a 100644
--- a/src/kudu/codegen/row_projector.cc
+++ b/src/kudu/codegen/row_projector.cc
@@ -380,7 +380,7 @@ struct DefaultEquals {
struct ColumnSchemaEqualsType {
bool operator()(const ColumnSchema& s1, const ColumnSchema& s2) {
- return s1.EqualsType(s2);
+ return s1.EqualsPhysicalType(s2);
}
};
@@ -418,7 +418,7 @@ bool ContainerEquals(const T& t1, const T& t2) {
// the actual dependency on column identification - which is the effect
// that those attributes have on the RowProjector's mapping (i.e., different
// names and IDs are ok, so long as the mapping is the same). Note that
-// key columns are not given any special meaning in projection. Types
+// key columns are not given any special meaning in projection. Physical types
// and nullability of columns must be exactly equal between the two
// schema pairs.
//
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/aa391149/src/kudu/common/schema.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/schema.h b/src/kudu/common/schema.h
index 0c16c94..bebaff5 100644
--- a/src/kudu/common/schema.h
+++ b/src/kudu/common/schema.h
@@ -195,6 +195,11 @@ class ColumnSchema {
return NULL;
}
+ bool EqualsPhysicalType(const ColumnSchema& other) const {
+ return is_nullable_ == other.is_nullable_ &&
+ type_info()->physical_type() == other.type_info()->physical_type();
+ }
+
bool EqualsType(const ColumnSchema &other) const {
return is_nullable_ == other.is_nullable_ &&
type_info()->type() == other.type_info()->type();
@@ -204,7 +209,7 @@ class ColumnSchema {
if (!EqualsType(other) || this->name_ != other.name_)
return false;
- // For Key comparison checking the defauls doesn't make sense,
+ // For Key comparison checking the defaults doesn't make sense,
// since we don't support them, for server vs user schema this comparison
// will always fail, since the user does not specify the defaults.
if (check_defaults) {
[3/3] incubator-kudu git commit: KUDU-1382 release notes for 0.8.0
Posted by ad...@apache.org.
KUDU-1382 release notes for 0.8.0
Also change ordered lists to bullet lists throughout
release notes.
Change-Id: I4e83f314cbb373f05b044bd46602713c7dbaee3e
Reviewed-on: http://gerrit.cloudera.org:8080/2655
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
(cherry picked from commit 959952a93c86292f45ded234065c75b430d42c4b)
Reviewed-on: http://gerrit.cloudera.org:8080/2689
Reviewed-by: Jean-Daniel Cryans
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/60c155cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/60c155cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/60c155cf
Branch: refs/heads/branch-0.8.x
Commit: 60c155cf07952ca90723f4cdeb5c3a785fefcd0a
Parents: aa39114
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Mon Mar 28 11:28:41 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Mar 31 23:56:32 2016 +0000
----------------------------------------------------------------------
docs/installation.adoc | 19 +++----
docs/release_notes.adoc | 127 +++++++++++++++++++++++++++++++++++--------
2 files changed, 111 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/60c155cf/docs/installation.adoc
----------------------------------------------------------------------
diff --git a/docs/installation.adoc b/docs/installation.adoc
index a9d890b..4d47bfa 100644
--- a/docs/installation.adoc
+++ b/docs/installation.adoc
@@ -621,15 +621,14 @@ in `java/kudu-client/target/apidocs/index.html`.
// end::view_api[]
[[upgrade]]
-== Upgrade from 0.6.0 to 0.7.0
+== Upgrade from 0.7.1 to 0.8.0
Before upgrading, see <<client_compatibility>> and <<api_compatibility>>.
-To upgrade from Kudu 0.6.0 to 0.7.0, perform the following high-level steps, which
+To upgrade from Kudu 0.7.1 to 0.8.0, perform the following high-level steps, which
are detailed in <<upgrade_procedure>>:
. Shut down all Kudu services.
-. Install the new Kudu packages or parcels, or install Kudu 0.7.0 from source.
-. Ubuntu only: re-download the repository list file. See <<kudu_package_locations>>.
+. Install the new Kudu packages or parcels, or install Kudu 0.8.0 from source.
. Restart all Kudu services.
It is technically possible to upgrade Kudu using rolling restarts, but it has not
@@ -641,18 +640,16 @@ from the previous latest version to the newest.
[[client_compatibility]]
=== Client compatibility
-Kudu 0.7.0 maintains wire compatibility with previous versions. This means that a Kudu 0.7.0
-client can communicate with a Kudu 0.6.0 cluster, and vice versa. For that reason,
-you do not need to upgrade client JARs at the same time the cluster is upgraded.
-
-The same wire compatibility guarantees apply to the Impala_Kudu fork that was released
-with Kudu 0.5.0.
+Masters and tablet servers should be upgraded before clients are upgraded. For specific
+information about client compatibility, see the
+link:release_notes.html#rn_0.8.0_incompatible_changes[Incompatible Changes] section
+of the release notes.
[[api_compatibility]]
=== API Compatibility
-The Kudu 0.7.0 client API is compatible with Kudu 0.6.0.
+The Kudu 0.8.0 client API is compatible with Kudu 0.7.1.
[[upgrade_procedure]]
=== Upgrade procedure
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/60c155cf/docs/release_notes.adoc
----------------------------------------------------------------------
diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc
index 4697516..16f7456 100644
--- a/docs/release_notes.adoc
+++ b/docs/release_notes.adoc
@@ -53,6 +53,85 @@ By combining all of these properties, Kudu targets support for families of
applications that are difficult or impossible to implement on current-generation
Hadoop storage technologies.
+[[rn_0.8.0]]
+=== Release notes specific to 0.8.0
+
+Kudu 0.8.0 delivers incremental features, improvements, and bug fixes over the previous versions.
+
+See also +++<a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20KUDU%20AND%20status%20%3D%20Resolved
+%20AND%20fixVersion%20%3D%200.8.0">JIRAs resolved
+for Kudu 0.8.0</a>+++ and +++<a href="https://github.com/apache/incubator-kudu/compare/0.7.1...0.8.0">Git
+changes between 0.7.1 and 0.8.0</a>+++.
+
+To upgrade to Kudu 0.8.0, see link:installation.html#upgrade[Upgrade from 0.7.1 to 0.8.0].
+
+[[rn_0.8.0_incompatible_changes]]
+==== Incompatible changes
+
+- 0.8.0 clients are not fully compatible with servers running Kudu 0.7.1 or lower.
+In particular, scans that specify column predicates will fail. To work around this
+issue, upgrade all Kudu servers before upgrading clients.
+
+[[rn_0.8.0_new_features]]
+==== New features
+
+- link:https://issues.apache.org/jira/browse/KUDU-431[KUDU-431] A simple Flume
+ sink has been implemented.
+
+[[rn_0.8.0_improvements]]
+==== Improvements
+
+- link:https://issues.apache.org/jira/browse/KUDU-839[KUDU-839] Java RowError now uses an enum error code.
+
+- link:http://gerrit.cloudera.org:8080/#/c/2138/[Gerrit 2138] The handling of
+ column predicates has been re-implemented in the server and clients.
+
+- link:https://issues.apache.org/jira/browse/KUDU-1379[KUDU-1379] Partition pruning
+ has been implemented for C++ clients (but not yet for the Java client). This feature
+ allows you to avoid reading a tablet if you know it does not serve the row keys you are querying.
+
+- link:http://gerrit.cloudera.org:8080/#/c/2641[Gerrit 2641] Kudu now uses
+ `earliest-deadline-first` RPC scheduling and rejection. This changes the behavior
+ of the RPC service queue to prevent unfairness when processing a backlog of RPC
+ threads and to increase the likelihood that an RPC will be processed before it
+ can time out.
+
+
+[[rn_0.8.0_fixed_issues]]
+==== Fixed Issues
+
+- link:https://issues.cloudera.org/browse/KUDU-1337[KUDU-1337] Tablets from tables
+ that were deleted might be unnecessarily re-bootstrapped when the leader gets the
+ notification to delete itself after the replicas do.
+
+- link:https://issues.cloudera.org/browse/KUDU-969[KUDU-969] If a tablet server
+ shuts down while compacting a rowset and receiving updates for it, it might immediately
+ crash upon restart while bootstrapping that rowset's tablet.
+
+- link:https://issues.cloudera.org/browse/KUDU-1354[KUDU-1354] Due to a bug in Kudu's
+ MVCC implementation where row locks were released before the MVCC commit happened,
+ flushed data would include out-of-order transactions, triggering a crash on the
+ next compaction.
+
+- link:https://issues.apache.org/jira/browse/KUDU-1322[KUDU-1322] The C++ client
+ now retries write operations if the tablet it is trying to reach has already been
+ deleted.
+
+- link:http://gerrit.cloudera.org:8080/#/c/2571/[Gerrit 2571] Due to a bug in the
+ Java client, users were unable to close the `kudu-spark` shell because of
+ lingering non-daemon threads.
+
+[[rn_0.8.0_changes]]
+==== Other noteworthy changes
+
+- link:http://gerrit.cloudera.org:8080/#/c/2239/[Gerrit 2239] The concept of "feature flags"
+ was introduced in order to manage compability between different
+ Kudu versions. One case where this is helpful is if a newer client attempts to use
+ a feature unsupported by the currently-running tablet server. Rather than receiving
+ a cryptic error, the user gets an error message that is easier to interpret.
+ This is an internal change for Kudu system developers and requires no action by
+ users of the clients or API.
+
[[rn_0.7.1]]
=== Release notes specific to 0.7.1
@@ -62,26 +141,26 @@ Kudu 0.7.1 is a bug fix release for 0.7.0.
==== Fixed Issues
-. https://issues.apache.org/jira/browse/KUDU-1325[KUDU-1325] fixes a tablet server crash that could
+- https://issues.apache.org/jira/browse/KUDU-1325[KUDU-1325] fixes a tablet server crash that could
occur during table deletion. In some cases, while a table was being deleted, other replicas would
attempt to re-replicate tablets to servers that had already processed the deletion. This could
trigger a race condition that caused a crash.
-. https://issues.apache.org/jira/browse/KUDU-1341[KUDU-1341] fixes a potential data corruption and
+- https://issues.apache.org/jira/browse/KUDU-1341[KUDU-1341] fixes a potential data corruption and
crash that could happen shortly after tablet server restarts in workloads that repeatedly delete
and re-insert rows with the same primary key. In most cases, this corruption affected only a single
replica and could be repaired by re-replicating from another.
-. https://issues.apache.org/jira/browse/KUDU-1343[KUDU-1343] fixes a bug in the Java client that
+- https://issues.apache.org/jira/browse/KUDU-1343[KUDU-1343] fixes a bug in the Java client that
occurs when a scanner has to scan multiple batches from one tablet and then start scanning from
another. In particular, this would affect any scans using the Java client that read large numbers
of rows from multi-tablet tables.
-. https://issues.apache.org/jira/browse/KUDU-1345[KUDU-1345] fixes a bug where in some cases the
+- https://issues.apache.org/jira/browse/KUDU-1345[KUDU-1345] fixes a bug where in some cases the
hybrid clock could jump backwards, resulting in a crash followed by an inability to
restart the affected tablet server.
-. https://issues.apache.org/jira/browse/KUDU-1360[KUDU-1360] fixes a bug in the kudu-spark module
+- https://issues.apache.org/jira/browse/KUDU-1360[KUDU-1360] fixes a bug in the kudu-spark module
which prevented reading rows with `NULL` values.
[[rn_0.7.0]]
@@ -100,19 +179,19 @@ The upgrade instructions can be found at link:installation.html#upgrade[Upgrade
[[rn_0.7.0_incompatible_changes]]
==== Incompatible changes
-. The C++ client includes a new API, `KuduScanBatch`, which performs better when a
+- The C++ client includes a new API, `KuduScanBatch`, which performs better when a
large number of small rows are returned in a batch. The old API of `vector<KuduRowResult>`
is deprecated.
+
NOTE: This change is API-compatible but *not* ABI-compatible.
-. The default replication factor has been changed from 1 to 3. Existing tables will
+- The default replication factor has been changed from 1 to 3. Existing tables will
continue to use the replication factor they were created with. Applications that create
tables may not work properly if they assume a replication factor of 1 and fewer than
3 replicas are available. To use the previous default replication factor, start the
master with the configuration flag `--default_num_replicas=1`.
-. The Python client has been completely rewritten, with a focus on improving code
+- The Python client has been completely rewritten, with a focus on improving code
quality and testing. The read path (scanners) has been improved by adding many of
the features already supported by the C++ and Java clients. The Python client is no
longer considered experimental.
@@ -120,54 +199,54 @@ longer considered experimental.
[[rn_0.7.0_new_features]]
==== New features
-. With the goal of Spark integration in mind, a new `kuduRDD` API has been added,
+- With the goal of Spark integration in mind, a new `kuduRDD` API has been added,
which wraps `newAPIHadoopRDD` and includes a default source for Spark SQL.
[[rn_0.7.0_improvements]]
==== Improvements
-. The Java client includes new methods `countPendingErrors()` and
+- The Java client includes new methods `countPendingErrors()` and
`getPendingErrors()` on `KuduSession`. These methods allow you to count and
retrieve outstanding row errors when configuring sessions with `AUTO_FLUSH_BACKGROUND`.
-. New server-level metrics allow you to monitor CPU usage and context switching.
+- New server-level metrics allow you to monitor CPU usage and context switching.
-. Kudu now builds on RHEL 7, CentOS 7, and SLES 12. Extra instructions are included
+- Kudu now builds on RHEL 7, CentOS 7, and SLES 12. Extra instructions are included
for SLES 12.
[[rn_0.7.0_fixed_issues]]
==== Fixed Issues
-. https://issues.cloudera.org/browse/KUDU-1288[KUDU-1288] fixes a severe file descriptor
+- https://issues.cloudera.org/browse/KUDU-1288[KUDU-1288] fixes a severe file descriptor
leak, which could previously only be resolved by restarting the tablet server.
-. https://issues.cloudera.org/browse/KUDU-1250[KUDU-1250] fixes a hang in the Java
+- https://issues.cloudera.org/browse/KUDU-1250[KUDU-1250] fixes a hang in the Java
client when processing an in-flight batch and the previous batch encountered an error.
[[rn_0.7.0_changes]]
==== Other noteworthy changes
-. The file block manager's performance was improved, but it is still not recommended for
+- The file block manager's performance was improved, but it is still not recommended for
real-world use.
-. The master now attempts to spread tablets more evenly across the cluster during
+- The master now attempts to spread tablets more evenly across the cluster during
table creation. This has no impact on existing tables, but will improve the speed
at which under-replicated tabletsare re-replicated after a tablet server failure.
-. All licensing documents have been modified to adhere to ASF guidelines.
+- All licensing documents have been modified to adhere to ASF guidelines.
-. Kudu now requires an out-of-tree build directory. Review the build instructions
+- Kudu now requires an out-of-tree build directory. Review the build instructions
for additional information.
-. The `C++` client library is now explicitly built against the
+- The `C++` client library is now explicitly built against the
link:https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html[old gcc5 ABI].
If you use gcc5 to build a Kudu application, your application must use the old ABI
as well. This is typically achieved by defining the `_GLIBCXX_USE_CXX11_ABI` macro
at compile-time when building your application. For more information, see the
previous link and link:http://developerblog.redhat.com/2015/02/05/gcc5-and-the-c11-abi/.
-. The Python client is no longer considered experimental.
+- The Python client is no longer considered experimental.
==== Limitations
@@ -185,15 +264,15 @@ Trusty, and SLES 12. Other operating systems may work but have not been tested.
The 0.6.0 release contains incremental improvements and bug fixes. The most notable
changes are:
-. The Java client's CreateTableBuilder and AlterTableBuilder classes have been renamed
+- The Java client's CreateTableBuilder and AlterTableBuilder classes have been renamed
to CreateTableOptions and AlterTableOptions. Their methods now also return `this` objects,
allowing them to be used as builders.
-. The Java client's AbstractKuduScannerBuilder#maxNumBytes() setter is now called
+- The Java client's AbstractKuduScannerBuilder#maxNumBytes() setter is now called
batchSizeBytes as is the corresponding property in AsyncKuduScanner. This makes it
consistent with the C++ client.
-. The "kudu-admin" tool can now list and delete tables via its new subcommands
+- The "kudu-admin" tool can now list and delete tables via its new subcommands
"list_tables" and "delete_table <table_name>".
-. OSX is now supported for single-host development. Please consult its specific installation
+- OSX is now supported for single-host development. Please consult its specific installation
instructions in link:installation.html#osx_from_source[OS X].
==== Limitations