You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/12/12 19:41:43 UTC
[1/2] kudu git commit: [spark] KUDU-1641 Pushdown SparkSQL In
predicates
Repository: kudu
Updated Branches:
refs/heads/master ee4672bbe -> 11f4121a9
[spark] KUDU-1641 Pushdown SparkSQL In predicates
This patch adds support for pushing down "IN (values)" predicates to Kudu
using the in list predicate.
Change-Id: Ia76dcbc0b5c5dcb2d28b468d8ed305b9f8eb8b28
Reviewed-on: http://gerrit.cloudera.org:8080/5425
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6f7df351
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6f7df351
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6f7df351
Branch: refs/heads/master
Commit: 6f7df3513c38023906fe744616ac2dacd9ef0524
Parents: ee4672b
Author: Will Berkeley <wd...@gmail.com>
Authored: Thu Dec 8 15:20:38 2016 -0500
Committer: Dan Burkert <da...@apache.org>
Committed: Mon Dec 12 18:13:35 2016 +0000
----------------------------------------------------------------------
.../apache/kudu/spark/kudu/DefaultSource.scala | 16 ++++++++++-
.../kudu/spark/kudu/DefaultSourceTest.scala | 30 ++++++++++++++++++++
2 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/6f7df351/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 87999ba..1201591 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -169,6 +169,8 @@ with InsertableRelation {
Array(comparisonPredicate(column, ComparisonOp.LESS, value))
case LessThanOrEqual(column, value) =>
Array(comparisonPredicate(column, ComparisonOp.LESS_EQUAL, value))
+ case In(column, values) =>
+ Array(inListPredicate(column, values))
case And(left, right) => filterToPredicate(left) ++ filterToPredicate(right)
case _ => Array()
}
@@ -201,6 +203,17 @@ with InsertableRelation {
}
/**
+ * Creates a new in list predicate for the column and values.
+ *
+ * @param column the column name
+ * @param values the values
+ * @return the in list predicate
+ */
+ private def inListPredicate(column: String, values: Array[Any]): KuduPredicate = {
+ KuduPredicate.newInListPredicate(table.getSchema.getColumn(column), values.toList.asJava)
+ }
+
+ /**
* Writes data into an existing Kudu table.
*
* If the `kudu.operation` parameter is set, the data will use that operation
@@ -247,7 +260,8 @@ private[spark] object KuduRelation {
| GreaterThan(_, _)
| GreaterThanOrEqual(_, _)
| LessThan(_, _)
- | LessThanOrEqual(_, _) => true
+ | LessThanOrEqual(_, _)
+ | In(_, _) => true
case And(left, right) => supportsFilter(left) && supportsFilter(right)
case _ => false
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/6f7df351/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index 4cb15db..894a5d5 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -272,6 +272,36 @@ class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
assert(results.get(0).getInt(0).equals(2))
}
+ test("Test basic SparkSQL with in list predicate") {
+ val keys = Array(1, 5, 7)
+ val results = sqlContext.sql(s"SELECT key FROM $tableName where key in (${keys.mkString(", ")})").collectAsList()
+ assert(results.size() == keys.length)
+ keys.zipWithIndex.foreach { case (v, i) =>
+ assert(results.get(i).size.equals(1))
+ assert(results.get(i).getInt(0).equals(v))
+ }
+ }
+
+ test("Test basic SparkSQL with in list predicate on string") {
+ val keys = Array(1, 4, 6)
+ val results = sqlContext.sql(s"SELECT key FROM $tableName where c2_s in (${keys.mkString("'", "', '", "'")})").collectAsList()
+ assert(results.size() == keys.count(_ % 2 == 0))
+ keys.filter(_ % 2 == 0).zipWithIndex.foreach { case (v, i) =>
+ assert(results.get(i).size.equals(1))
+ assert(results.get(i).getInt(0).equals(v))
+ }
+ }
+
+ test("Test basic SparkSQL with in list and comparison predicate") {
+ val keys = Array(1, 5, 7)
+ val results = sqlContext.sql(s"SELECT key FROM $tableName where key>2 and key in (${keys.mkString(", ")})").collectAsList()
+ assert(results.size() == keys.count(_>2))
+ keys.filter(_>2).zipWithIndex.foreach { case (v, i) =>
+ assert(results.get(i).size.equals(1))
+ assert(results.get(i).getInt(0).equals(v))
+ }
+ }
+
test("Test basic SparkSQL with two predicates negative") {
val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=1 and c2_s='2'").collectAsList()
assert(results.size() == 0)
[2/2] kudu git commit: KUDU-1805: Properly qualify classes in
generated svc headers
Posted by ad...@apache.org.
KUDU-1805: Properly qualify classes in generated svc headers
Change-Id: Id67c60477133699959c0e1004e69871e20657fc9
Reviewed-on: http://gerrit.cloudera.org:8080/5478
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/11f4121a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/11f4121a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/11f4121a
Branch: refs/heads/master
Commit: 11f4121a93f0051cce588e145bf3aef68eda44c4
Parents: 6f7df35
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Dec 12 11:05:06 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Mon Dec 12 19:41:15 2016 +0000
----------------------------------------------------------------------
src/kudu/rpc/protoc-gen-krpc.cc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/11f4121a/src/kudu/rpc/protoc-gen-krpc.cc
----------------------------------------------------------------------
diff --git a/src/kudu/rpc/protoc-gen-krpc.cc b/src/kudu/rpc/protoc-gen-krpc.cc
index 3294d65..d935a00 100644
--- a/src/kudu/rpc/protoc-gen-krpc.cc
+++ b/src/kudu/rpc/protoc-gen-krpc.cc
@@ -359,8 +359,8 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
Print(printer, *subs,
"class $service_name$If : public ::kudu::rpc::GeneratedServiceIf {\n"
" public:\n"
- " explicit $service_name$If(const scoped_refptr<MetricEntity>& entity,"
- " const scoped_refptr<rpc::ResultTracker>& result_tracker);\n"
+ " explicit $service_name$If(const scoped_refptr<::kudu::MetricEntity>& entity,"
+ " const scoped_refptr<::kudu::rpc::ResultTracker>& result_tracker);\n"
" virtual ~$service_name$If();\n"
" std::string service_name() const override;\n"
" static std::string static_service_name();\n"