You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/09/25 06:28:55 UTC
[3/3] incubator-impala git commit: IMPALA-1767 Adds predicate to test
boolean values true, false, unknown.
IMPALA-1767 Adds predicate to test boolean values true, false, unknown.
Adds a new expression to represent the following boolean predicate:
<expr> IS [NOT] (TRUE | FALSE | UNKNOWN)
The expression is expanded in the parser to istrue/false for the checks
against true and false respectively and to isnull for the check against
unknown. Compared to the other approaches (rewrites, extended backend expr),
this change is the simplest. Main downside is that error messages are
in terms of the lowered expression.
Testing:
- fe: parser, tosql, analyze exprs
- e2e: query exprs
Change-Id: I9d5fba65ef6c87dfc55a25d2c45246f74eb48c40
Reviewed-on: http://gerrit.cloudera.org:8080/8122
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/64692081
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/64692081
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/64692081
Branch: refs/heads/master
Commit: 646920810f581698c5150553d6bccc0f82c0fe6b
Parents: d53f43b
Author: Vuk Ercegovac <ve...@cloudera.com>
Authored: Thu Sep 21 13:47:37 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Sep 23 04:33:20 2017 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 42 +++++++++++++++++++
fe/src/main/cup/sql-parser.cup | 34 ++++++++++++++-
.../impala/analysis/AnalyzeExprsTest.java | 40 ++++++++++++++++++
.../org/apache/impala/analysis/ParserTest.java | 22 +++++++---
.../org/apache/impala/analysis/ToSqlTest.java | 10 +++++
.../queries/QueryTest/exprs.test | 44 ++++++++++++++++++++
6 files changed, 184 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index e4d633d..6a63ad5 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -2548,6 +2548,48 @@ TEST_F(ExprTest, IsNullPredicate) {
TestValue("NULL IS NOT NULL", TYPE_BOOLEAN, false);
}
+TEST_F(ExprTest, BoolTestExpr) {
+ // Tests against constants.
+ TestValue("TRUE IS TRUE", TYPE_BOOLEAN, true);
+ TestValue("TRUE IS FALSE", TYPE_BOOLEAN, false);
+ TestValue("TRUE IS UNKNOWN", TYPE_BOOLEAN, false);
+ TestValue("TRUE IS NOT TRUE", TYPE_BOOLEAN, false);
+ TestValue("TRUE IS NOT FALSE", TYPE_BOOLEAN, true);
+ TestValue("TRUE IS NOT UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("FALSE IS TRUE", TYPE_BOOLEAN, false);
+ TestValue("FALSE IS FALSE", TYPE_BOOLEAN, true);
+ TestValue("FALSE IS UNKNOWN", TYPE_BOOLEAN, false);
+ TestValue("FALSE IS NOT TRUE", TYPE_BOOLEAN, true);
+ TestValue("FALSE IS NOT FALSE", TYPE_BOOLEAN, false);
+ TestValue("FALSE IS NOT UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("NULL IS TRUE", TYPE_BOOLEAN, false);
+ TestValue("NULL IS FALSE", TYPE_BOOLEAN, false);
+ TestValue("NULL IS UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("NULL IS NOT TRUE", TYPE_BOOLEAN, true);
+ TestValue("NULL IS NOT FALSE", TYPE_BOOLEAN, true);
+ TestValue("NULL IS NOT UNKNOWN", TYPE_BOOLEAN, false);
+
+ // Tests against expressions
+ TestValue("(2>1) IS TRUE", TYPE_BOOLEAN, true);
+ TestValue("(2>1) IS FALSE", TYPE_BOOLEAN, false);
+ TestValue("(2>1) IS UNKNOWN", TYPE_BOOLEAN, false);
+ TestValue("(2>1) IS NOT TRUE", TYPE_BOOLEAN, false);
+ TestValue("(2>1) IS NOT FALSE", TYPE_BOOLEAN, true);
+ TestValue("(2>1) IS NOT UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("(1>2) IS TRUE", TYPE_BOOLEAN, false);
+ TestValue("(1>2) IS FALSE", TYPE_BOOLEAN, true);
+ TestValue("(1>2) IS UNKNOWN", TYPE_BOOLEAN, false);
+ TestValue("(1>2) IS NOT TRUE", TYPE_BOOLEAN, true);
+ TestValue("(1>2) IS NOT FALSE", TYPE_BOOLEAN, false);
+ TestValue("(1>2) IS NOT UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("(NULL = 1) IS TRUE", TYPE_BOOLEAN, false);
+ TestValue("(NULL = 1) IS FALSE", TYPE_BOOLEAN, false);
+ TestValue("(NULL = 1) IS UNKNOWN", TYPE_BOOLEAN, true);
+ TestValue("(NULL = 1) IS NOT TRUE", TYPE_BOOLEAN, true);
+ TestValue("(NULL = 1) IS NOT FALSE", TYPE_BOOLEAN, true);
+ TestValue("(NULL = 1) IS NOT UNKNOWN", TYPE_BOOLEAN, false);
+}
+
TEST_F(ExprTest, LikePredicate) {
TestValue("'a' LIKE '%a%'", TYPE_BOOLEAN, true);
TestValue("'a' LIKE '%abcde'", TYPE_BOOLEAN, false);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 0eaa68e..7c0a794 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -339,8 +339,9 @@ nonterminal ArrayList<String> opt_ident_list, opt_sort_cols;
nonterminal TableName table_name;
nonterminal FunctionName function_name;
nonterminal Expr where_clause;
-nonterminal Predicate predicate, between_predicate, comparison_predicate,
- compound_predicate, in_predicate, like_predicate, exists_predicate;
+nonterminal Expr predicate, bool_test_expr;
+nonterminal Predicate between_predicate, comparison_predicate, compound_predicate,
+ in_predicate, like_predicate, exists_predicate;
nonterminal ArrayList<Expr> group_by_clause, opt_partition_by_clause;
nonterminal Expr having_clause;
nonterminal ArrayList<OrderByElement> order_by_elements, opt_order_by_clause;
@@ -493,6 +494,7 @@ nonterminal Boolean server_ident;
nonterminal Boolean source_ident;
nonterminal Boolean sources_ident;
nonterminal Boolean uri_ident;
+nonterminal Boolean unknown_ident;
// For Create/Drop/Show function ddl
nonterminal FunctionArgs function_def_args;
@@ -1678,6 +1680,16 @@ option_ident ::=
:}
;
+unknown_ident ::=
+ IDENT:ident
+ {:
+ if (!ident.toUpperCase().equals("UNKNOWN")) {
+ parser.parseError("identifier", SqlParserSymbols.IDENT, "UNKNOWN");
+ }
+ RESULT = true;
+ :}
+ ;
+
view_column_defs ::=
LPAREN view_column_def_list:view_col_defs RPAREN
{: RESULT = view_col_defs; :}
@@ -2933,6 +2945,8 @@ predicate ::=
{: RESULT = p; :}
| like_predicate:p
{: RESULT = p; :}
+ | bool_test_expr:e
+ {: RESULT = e; :}
| LPAREN predicate:p RPAREN
{:
p.setPrintSqlInParens(true);
@@ -3017,6 +3031,22 @@ in_predicate ::=
{: RESULT = new InPredicate(e, s, true); :}
;
+// Boolean test expression: <expr> IS [NOT] (TRUE | FALSE | UNKNOWN)
+bool_test_expr ::=
+ expr:e KW_IS KW_TRUE
+ {: RESULT = new FunctionCallExpr("istrue", Lists.newArrayList(e)); :}
+ | expr:e KW_IS KW_NOT KW_TRUE
+ {: RESULT = new FunctionCallExpr("isnottrue", Lists.newArrayList(e)); :}
+ | expr:e KW_IS KW_FALSE
+ {: RESULT = new FunctionCallExpr("isfalse", Lists.newArrayList(e)); :}
+ | expr:e KW_IS KW_NOT KW_FALSE
+ {: RESULT = new FunctionCallExpr("isnotfalse", Lists.newArrayList(e)); :}
+ | expr:e KW_IS unknown_ident
+ {: RESULT = new IsNullPredicate(e, false); :}
+ | expr:e KW_IS KW_NOT unknown_ident
+ {: RESULT = new IsNullPredicate(e, true); :}
+ ;
+
subquery ::=
LPAREN subquery:s RPAREN
{: RESULT = s; :}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index e393f65..8891fee 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -613,6 +613,7 @@ public class AnalyzeExprsTest extends AnalyzerTest {
AnalyzesOk("select * from functional.alltypes where int_col is null");
AnalyzesOk("select * from functional.alltypes where string_col is not null");
AnalyzesOk("select * from functional.alltypes where null is not null");
+
AnalysisError("select 1 from functional.allcomplextypes where int_map_col is null",
"IS NULL predicate does not support complex types: int_map_col IS NULL");
AnalysisError("select * from functional.allcomplextypes where complex_struct_col " +
@@ -624,6 +625,45 @@ public class AnalyzeExprsTest extends AnalyzerTest {
}
@Test
+ public void TestBoolTestExpression() throws AnalysisException {
+ String[] rhsOptions = new String[] {"true", "false", "unknown"};
+ String[] lhsOptions = new String[] {
+ "bool_col", // column reference
+ "1>1", // boolean expression
+ "istrue(false)", // function
+ "(1>1 is true)" // nested expression
+ };
+
+ // Bool test in both the select and where clauses. Includes negated IS clauses.
+ for (String rhsVal : rhsOptions) {
+ String template = "select (%s is %s) from functional.alltypes where (%s is %s)";
+ String lhsVal = rhsVal == "unknown" ? "null" : rhsVal;
+ String negatedRhsVal = "not " + rhsVal;
+
+ // Tests for lhs being one of true, false or null.
+ AnalyzesOk(String.format(template, lhsVal, rhsVal, lhsVal, rhsVal));
+ AnalyzesOk(String.format(template, lhsVal, negatedRhsVal, lhsVal, negatedRhsVal));
+
+ // Tests for lhs being one lhsOptions expressions.
+ for (String lhs : lhsOptions) {
+ AnalyzesOk(String.format(template, lhs, rhsVal, lhs, rhsVal));
+ AnalyzesOk(String.format(template, lhs, negatedRhsVal, lhs, negatedRhsVal));
+ }
+ }
+
+ AnalysisError("select ('foo' is true)",
+ "No matching function with signature: istrue(STRING).");
+ AnalysisError("select (10 is true)",
+ "No matching function with signature: istrue(TINYINT).");
+ AnalysisError("select (10 is not true)",
+ "No matching function with signature: isnottrue(TINYINT).");
+ AnalysisError("select (10 is false)",
+ "No matching function with signature: isfalse(TINYINT).");
+ AnalysisError("select (10 is not false)",
+ "No matching function with signature: isnotfalse(TINYINT).");
+ }
+
+ @Test
public void TestBetweenPredicates() throws AnalysisException {
AnalyzesOk("select * from functional.alltypes " +
"where tinyint_col between smallint_col and int_col");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index d6d808f..1f9a6af 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -1446,17 +1446,27 @@ public class ParserTest extends FrontendTestBase {
operations.add("regexp");
operations.add("iregexp");
- for (String lop: operands_) {
- for (String rop: operands_) {
+ ArrayList<String> boolTestVals = new ArrayList<String>();
+ boolTestVals.add("null");
+ boolTestVals.add("unknown");
+ boolTestVals.add("true");
+ boolTestVals.add("false");
+
+ for (String lop : operands_) {
+ for (String rop : operands_) {
for (String op : operations) {
String expr = String.format("%s %s %s", lop, op.toString(), rop);
ParsesOk(String.format("select %s from t where %s", expr, expr));
}
}
- String isNullExr = String.format("%s is null", lop);
- String isNotNullExr = String.format("%s is not null", lop);
- ParsesOk(String.format("select %s from t where %s", isNullExr, isNullExr));
- ParsesOk(String.format("select %s from t where %s", isNotNullExr, isNotNullExr));
+ for (String val : boolTestVals) {
+ String isExpr = String.format("%s is %s", lop, val);
+ String isNotExpr = String.format("%s is not %s", lop, val);
+ ParsesOk(String.format("select %s from t where %s", isExpr, isExpr));
+ ParsesOk(String.format("select %s from t where %s", isNotExpr, isNotExpr));
+ }
+ ParserError(String.format("select %s is nonsense", lop));
+ ParserError(String.format("select %s is not nonsense", lop));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index 4cd8934..d70d7ee 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -1254,6 +1254,16 @@ public class ToSqlTest extends FrontendTestBase {
// IsNullPredicate.
testToSql("select 5 is null, (5 is null), 10 is not null, (10 is not null)",
"SELECT 5 IS NULL, (5 IS NULL), 10 IS NOT NULL, (10 IS NOT NULL)");
+ // Boolean test expression (expanded to istrue/false).
+ testToSql("select (true is true)", "SELECT (istrue(TRUE))");
+ testToSql("select (true is not true)", "SELECT (isnottrue(TRUE))");
+ testToSql("select (true is false)", "SELECT (isfalse(TRUE))");
+ testToSql("select (true is unknown)", "SELECT (TRUE IS NULL)");
+ testToSql("select (true is not unknown)", "SELECT (TRUE IS NOT NULL)");
+ testToSql("select not(true is true)", "SELECT NOT (istrue(TRUE))");
+ testToSql("select (false is false)", "SELECT (isfalse(FALSE))");
+ testToSql("select (null is unknown)", "SELECT (NULL IS NULL)");
+ testToSql("select (1 > 1 is true is unknown)", "SELECT (istrue(1 > 1) IS NULL)");
// LikePredicate.
testToSql("select 'a' LIKE '%b.', ('a' LIKE '%b.'), " +
"'a' ILIKE '%b.', ('a' ILIKE '%b.'), " +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64692081/testdata/workloads/functional-query/queries/QueryTest/exprs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 92854b1..811a169 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -71,6 +71,50 @@ select count(*) from alltypesagg where tinyint_col is not null
bigint
====
---- QUERY
+# boolean test: IS [NOT] (TRUE | FALSE | UNKNOWN)
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is unknown;
+---- RESULTS
+20
+---- TYPES
+bigint
+====
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is true;
+---- RESULTS
+1080
+---- TYPES
+bigint
+====
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is false;
+---- RESULTS
+9900
+---- TYPES
+bigint
+====
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is not unknown;
+---- RESULTS
+10980
+---- TYPES
+bigint
+====
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is not true;
+---- RESULTS
+9920
+---- TYPES
+bigint
+====
+---- QUERY
+select count(*) from alltypesagg where (int_col < 100) is not false;
+---- RESULTS
+1100
+---- TYPES
+bigint
+====
+---- QUERY
# =
select count(*) from alltypesagg where tinyint_col = 1
---- RESULTS