You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/04/30 21:10:18 UTC
[3/6] kudu git commit: KUDU-2399: Support IS NULL / IS NOT NULL
predicates in Python
KUDU-2399: Support IS NULL / IS NOT NULL predicates in Python
Add is_null and is_not_null predicates in kudu-python. Had to change
the KuduTestBase and TestScanner to default to None and not replace
None with 'nothing' so I could write tests for null predicates.
I also fixed the test_scan_rows_string_predicate_and_projection test
since it was missing the projections and failing.
Change-Id: I52a22d6d8a8fe9a6049270596eb131799c37f82f
Reviewed-on: http://gerrit.cloudera.org:8080/10072
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves <da...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3bf125f4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3bf125f4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3bf125f4
Branch: refs/heads/master
Commit: 3bf125f469a2929f7fe20e4e31f073376f37575d
Parents: 9173015
Author: Andy Stadtler <an...@phdata.io>
Authored: Sat Apr 14 14:06:29 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Mon Apr 30 19:58:11 2018 +0000
----------------------------------------------------------------------
python/kudu/client.pyx | 58 ++++++++++++++++++++++++++++++++++
python/kudu/libkudu_client.pxd | 2 ++
python/kudu/tests/common.py | 2 +-
python/kudu/tests/test_scanner.py | 33 +++++++++++++++++--
python/kudu/tests/util.py | 8 +----
5 files changed, 93 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/client.pyx
----------------------------------------------------------------------
diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx
index 799b44e..5997bee 100644
--- a/python/kudu/client.pyx
+++ b/python/kudu/client.pyx
@@ -995,6 +995,64 @@ cdef class Column:
return result
+ def is_not_null(Column self):
+ """
+ Creates a new IsNotNullPredicate for the Column which can be used for scanners
+ on this table.
+
+ Examples
+ --------
+ scanner.add_predicate(table[col_name].is_not_null())
+
+ Returns
+ -------
+ pred : Predicate
+ """
+
+ cdef:
+ KuduPredicate* pred
+ Slice col_name_slice
+ Predicate result
+ object _name = tobytes(self.name)
+
+ col_name_slice = Slice(<char*> _name, len(_name))
+
+ pred = (self.parent.ptr()
+ .NewIsNotNullPredicate(col_name_slice))
+
+ result = Predicate()
+ result.init(pred)
+
+ return result
+
+ def is_null(Column self):
+ """
+ Creates a new IsNullPredicate for the Column which can be used for scanners on this table.
+
+ Examples
+ --------
+ scanner.add_predicate(table[col_name].is_null())
+
+ Returns
+ -------
+ pred : Predicate
+ """
+
+ cdef:
+ KuduPredicate* pred
+ Slice col_name_slice
+ Predicate result
+ object _name = tobytes(self.name)
+
+ col_name_slice = Slice(<char*> _name, len(_name))
+
+ pred = (self.parent.ptr()
+ .NewIsNullPredicate(col_name_slice))
+
+ result = Predicate()
+ result.init(pred)
+
+ return result
class Partitioning(object):
""" Argument to Client.create_table(...) to describe table partitioning. """
http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/libkudu_client.pxd
----------------------------------------------------------------------
diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd
index a489784..5cd7731 100644
--- a/python/kudu/libkudu_client.pxd
+++ b/python/kudu/libkudu_client.pxd
@@ -609,6 +609,8 @@ cdef extern from "kudu/client/client.h" namespace "kudu::client" nogil:
C_KuduValue* value);
KuduPredicate* NewInListPredicate(const Slice& col_name,
vector[C_KuduValue*]* values)
+ KuduPredicate* NewIsNotNullPredicate(const Slice& col_name)
+ KuduPredicate* NewIsNullPredicate(const Slice& col_name)
KuduClient* client()
# const PartitionSchema& partition_schema()
http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/common.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py
index 9596eaa..e1629ea 100644
--- a/python/kudu/tests/common.py
+++ b/python/kudu/tests/common.py
@@ -114,7 +114,7 @@ class KuduTestBase(object):
builder = kudu.schema_builder()
builder.add_column('key', kudu.int32, nullable=False)
builder.add_column('int_val', kudu.int32)
- builder.add_column('string_val', kudu.string, default='nothing')
+ builder.add_column('string_val', kudu.string, default=None)
builder.add_column('unixtime_micros_val', kudu.unixtime_micros)
builder.set_primary_keys(['key'])
http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/test_scanner.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py
index 00fe7a8..a4d0091 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -70,10 +70,9 @@ class TestScanner(TestScanBase):
def test_scan_rows_string_predicate_and_projection(self):
scanner = self.table.scanner()
- scanner.set_projected_column_names(['key', 'string_val'])
-
sv = self.table['string_val']
+ scanner.set_projected_column_indexes([0, 2])
scanner.add_predicates([sv >= 'hello_20',
sv <= 'hello_22'])
@@ -102,6 +101,36 @@ class TestScanner(TestScanBase):
self.assertEqual(tuples, [self.tuples[98]])
+ def test_scan_rows_is_not_null_predicate(self):
+ """
+ Test scanner with an IsNotNull predicate on string_val column
+ """
+ pred = self.table['string_val'].is_not_null()
+ scanner = self.table.scanner()
+ scanner.add_predicate(pred)
+ scanner.open()
+
+ tuples = scanner.read_all_tuples()
+
+ rows = [i for i in range(100) if i % 2 == 0]
+
+ self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])
+
+ def test_scan_rows_is_null_predicate(self):
+ """
+ Test scanner with an IsNull predicate on string_val column
+ """
+ pred = self.table['string_val'].is_null()
+ scanner = self.table.scanner()
+ scanner.add_predicate(pred)
+ scanner.open()
+
+ tuples = scanner.read_all_tuples()
+
+ rows = [i for i in range(100) if i % 2 != 0]
+
+ self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])
+
def test_index_projection_with_schema(self):
scanner = self.table.scanner()
scanner.set_projected_column_indexes([0, 1])
http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/util.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/util.py b/python/kudu/tests/util.py
index a6779fc..6823b3a 100644
--- a/python/kudu/tests/util.py
+++ b/python/kudu/tests/util.py
@@ -56,13 +56,7 @@ class TestScanBase(KuduTestBase, unittest.TestCase):
session.flush()
self.table = table
- self.tuples = []
-
- # Replace missing values w/ defaults to test default values.
- for tuple in tuples:
- if tuple[2] == None:
- tuple = (tuple[0], tuple[1], 'nothing', tuple[3])
- self.tuples.append(tuple)
+ self.tuples = tuples
# Create table to test all types
# for various predicate tests