You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/09/27 22:57:11 UTC
[3/3] incubator-impala git commit: IMPALA-4142: qgen: Hive does not
support CTEs inside sub-query blocks
IMPALA-4142: qgen: Hive does not support CTEs inside sub-query blocks
Changes:
Hive does not support WITH clauses inside sub-query blocks, the DefaultProfile actually
already has an option to disable this called "use_nested_with" and the HiveProfile
returns False for this method. However, there is a bug in the usage of "use_nested_with";
it is not checked every time a sub-query is created. Specifically, it is not checked when
an in-line view is created, or when the query within a WITH clause is created (so it's
possible to create WITH ... AS (WITH ...)). This patch fixes both of these issues since
Hive throws a SQL compile exception for either case.
Testing:
* Added new unit tests
* All unit tests pass
* Tested locally against Hive
* Tested against Impala via Leopard
* Tested against Impala via the discrepancy checker
Change-Id: Ie585bbbd88a32f502457a4ca9f2f54c7e1ade354
Reviewed-on: http://gerrit.cloudera.org:8080/4534
Reviewed-by: Taras Bobrovytsky <tb...@cloudera.com>
Tested-by: Taras Bobrovytsky <tb...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b5b848f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b5b848f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b5b848f3
Branch: refs/heads/master
Commit: b5b848f373f90fef674e666c29ca54c6ccb54858
Parents: 0780d2c
Author: Sahil Takiar <st...@cloudera.com>
Authored: Thu Sep 15 15:22:25 2016 -0700
Committer: Taras Bobrovytsky <tb...@cloudera.com>
Committed: Tue Sep 27 22:47:21 2016 +0000
----------------------------------------------------------------------
tests/comparison/query_generator.py | 6 +-
tests/comparison/tests/test_use_nested_with.py | 68 +++++++++++++++++++++
2 files changed, 72 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b5b848f3/tests/comparison/query_generator.py
----------------------------------------------------------------------
diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py
index 4685934..a38aea4 100644
--- a/tests/comparison/query_generator.py
+++ b/tests/comparison/query_generator.py
@@ -249,7 +249,8 @@ class QueryGenerator(object):
with_clause_inline_views = TableExprList()
for with_clause_inline_view_idx \
in xrange(self.profile.get_with_clause_table_ref_count()):
- query = self.create_query(table_exprs)
+ query = self.create_query(table_exprs,
+ allow_with_clause=self.profile.use_nested_with())
with_clause_alias_count = getattr(self.root_query, 'with_clause_alias_count', 0) + 1
self.root_query.with_clause_alias_count = with_clause_alias_count
with_clause_inline_view = \
@@ -1116,7 +1117,8 @@ class QueryGenerator(object):
def _create_inline_view(self, table_exprs, required_type=None):
return InlineView(self.create_query(
- table_exprs, required_select_item_type=required_type))
+ table_exprs, required_select_item_type=required_type,
+ allow_with_clause=self.profile.use_nested_with()))
def _create_join_clause(self, from_clause, table_exprs):
join_type = self.profile.choose_join_type(JoinClause.JOINS_TYPES)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b5b848f3/tests/comparison/tests/test_use_nested_with.py
----------------------------------------------------------------------
diff --git a/tests/comparison/tests/test_use_nested_with.py b/tests/comparison/tests/test_use_nested_with.py
new file mode 100644
index 0000000..418fdd7
--- /dev/null
+++ b/tests/comparison/tests/test_use_nested_with.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from tests.comparison.common import TableExprList, Column, Table
+from tests.comparison.db_types import Int
+from tests.comparison.query_generator import QueryGenerator
+from tests.comparison.query_profile import DefaultProfile
+
+
+def test_use_nested_width_subquery():
+ """
+ Tests that setting DefaultProfile.use_nested_with to False works properly. Setting this
+ method to return False should prevent a WITH clause from being used inside a sub-query.
+ """
+
+ class MockQueryProfile(DefaultProfile):
+ """
+ A mock QueryProfile that sets use_nested_with to False and forces the
+ QueryGenerator to created nested queries.
+ """
+
+ def __init__(self):
+ super(MockQueryProfile, self).__init__()
+
+ # Force the QueryGenerator to create nested queries
+ self._bounds['MAX_NESTED_QUERY_COUNT'] = (4, 4)
+
+ # Force the QueryGenerator to use WITH clauses whenever possible
+ self._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = 1
+
+ # Force the QueryGenerator to create inline views whenever possible
+ self._probabilities['MISC']['INLINE_VIEW'] = 1
+
+ def use_nested_with(self):
+ return False
+
+ mock_query_gen = QueryGenerator(MockQueryProfile())
+
+ # Create two tables
+ table_expr_list = TableExprList()
+
+ right_table = Table("right_table")
+ right_table.add_col(Column("right_table", "right_col", Int))
+ table_expr_list.append(right_table)
+
+ left_table = Table("left_table")
+ left_table.add_col(Column("left_table", "left_col", Int))
+ table_expr_list.append(left_table)
+
+ # Check that each nested_query doesn't have a with clause
+ for nested_query in mock_query_gen.create_query(table_expr_list).nested_queries:
+ assert nested_query.with_clause is None
\ No newline at end of file