You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2018/05/30 18:09:36 UTC
[1/5] impala git commit: IMPALA-6338: Disable more flaky bloom filter
tests
Repository: impala
Updated Branches:
refs/heads/master e9bd917a2 -> 2f22a6f67
IMPALA-6338: Disable more flaky bloom filter tests
Until IMPALA-6338 is fixed, temporarily disable tests that are
affected by it - any test that has a 'limit' and relies on the
contents of the runtime profile. This patch disables the runtime
profile check for all such tests in bloom_filter.test
Change-Id: Ifc9da892efa3b27d63056ad8e3befac82808ffdb
Reviewed-on: http://gerrit.cloudera.org:8080/10530
Reviewed-by: Bikramjeet Vig <bi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ba7893cb
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ba7893cb
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ba7893cb
Branch: refs/heads/master
Commit: ba7893cb9e72752299743225139d052a5a0276a6
Parents: e9bd917
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Tue May 29 11:54:24 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed May 30 08:00:50 2018 +0000
----------------------------------------------------------------------
.../queries/QueryTest/bloom_filters.test | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/ba7893cb/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test b/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
index b8d896f..abc56e0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
@@ -33,8 +33,9 @@ select STRAIGHT_JOIN count(*) from (select * from tpch.lineitem a LIMIT 1) a
---- RESULTS
0
---- RUNTIME_PROFILE
-row_regex: .*1 of 1 Runtime Filter Published.*
-row_regex: .*Filter 0 \(64.00 KB\).*
+# TODO: reenable once IMPALA-6338 is fixed
+#row_regex: .*1 of 1 Runtime Filter Published.*
+#row_regex: .*Filter 0 \(64.00 KB\).*
====
---- QUERY
SET RUNTIME_FILTER_MODE=GLOBAL;
@@ -46,8 +47,9 @@ select STRAIGHT_JOIN count(*) from (select * from tpch.lineitem a LIMIT 1) a
---- RESULTS
0
---- RUNTIME_PROFILE
-row_regex: .*1 of 1 Runtime Filter Published.*
-row_regex: .*Filter 0 \(256.00 KB\).*
+# TODO: reenable once IMPALA-6338 is fixed
+#row_regex: .*1 of 1 Runtime Filter Published.*
+#row_regex: .*Filter 0 \(256.00 KB\).*
====
---- QUERY
SET RUNTIME_FILTER_MODE=GLOBAL;
@@ -59,8 +61,9 @@ select STRAIGHT_JOIN count(*) from (select * from tpch.lineitem a LIMIT 1) a
---- RESULTS
0
---- RUNTIME_PROFILE
-row_regex: .*1 of 1 Runtime Filter Published.*
-row_regex: .*Filter 0 \(512.00 KB\).*
+# TODO: reenable once IMPALA-6338 is fixed
+#row_regex: .*1 of 1 Runtime Filter Published.*
+#row_regex: .*Filter 0 \(512.00 KB\).*
====
---- QUERY
SET RUNTIME_FILTER_MODE=GLOBAL;
@@ -115,8 +118,9 @@ with l as (select * from tpch.lineitem UNION ALL select * from tpch.lineitem)
select STRAIGHT_JOIN count(*) from (select * from tpch.lineitem a LIMIT 1) a
join (select * from l LIMIT 1000000) b on a.l_orderkey = -b.l_orderkey;
---- RUNTIME_PROFILE
-row_regex: .*0 of 1 Runtime Filter Published.*
-row_regex: .*Filter 0 \(64.00 KB\).*
+# TODO: reenable once IMPALA-6338 is fixed
+#row_regex: .*0 of 1 Runtime Filter Published.*
+#row_regex: .*Filter 0 \(64.00 KB\).*
====
[4/5] impala git commit: [DOCS] A typo fix in runtime filtering doc
Posted by tm...@apache.org.
[DOCS] A typo fix in runtime filtering doc
Change-Id: I43006540ed24c957c9ad2ec54707bb9c678d6fb3
Reviewed-on: http://gerrit.cloudera.org:8080/10544
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e4c52e31
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e4c52e31
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e4c52e31
Branch: refs/heads/master
Commit: e4c52e31551333af0ef37367a33b5e89b9b74d2f
Parents: aba8fdf
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Wed May 30 09:53:08 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed May 30 17:13:35 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_runtime_filtering.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e4c52e31/docs/topics/impala_runtime_filtering.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_runtime_filtering.xml b/docs/topics/impala_runtime_filtering.xml
index ae0f1b4..05c3743 100644
--- a/docs/topics/impala_runtime_filtering.xml
+++ b/docs/topics/impala_runtime_filtering.xml
@@ -166,7 +166,7 @@ under the License.
<conbody>
<p>
The <term>filter</term> that is transmitted between plan fragments is essentially a list
- of values for join key columns. When this list is values is transmitted in time to a scan node,
+ of values for join key columns. When this list of values is transmitted in time to a scan node,
Impala can filter out non-matching values immediately after reading them, rather than transmitting
the raw data to another host to compare against the in-memory hash table on that host.
</p>
[5/5] impala git commit: [DOCS] Fixed an inconsistent table ordering
recommendations with STRAIGHT_JOIN
Posted by tm...@apache.org.
[DOCS] Fixed an inconsistent table ordering recommendations with STRAIGHT_JOIN
Change-Id: I9ff2245b6d8e3160a10aae64f622813a40121ee8
Reviewed-on: http://gerrit.cloudera.org:8080/10519
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2f22a6f6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2f22a6f6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2f22a6f6
Branch: refs/heads/master
Commit: 2f22a6f67ff363a0832a7ceee5d0020c8fd9b15a
Parents: e4c52e3
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Fri May 25 13:14:24 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed May 30 17:50:04 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_perf_joins.xml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/2f22a6f6/docs/topics/impala_perf_joins.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_perf_joins.xml b/docs/topics/impala_perf_joins.xml
index 05dadbb..60c7b62 100644
--- a/docs/topics/impala_perf_joins.xml
+++ b/docs/topics/impala_perf_joins.xml
@@ -85,13 +85,13 @@ under the License.
<li>
Join the next smallest table, then the next smallest, and so on.
</li>
+ </ul>
- <li>
+ <p>
For example, if you had tables <codeph>BIG</codeph>, <codeph>MEDIUM</codeph>, <codeph>SMALL</codeph>, and
<codeph>TINY</codeph>, the logical join order to try would be <codeph>BIG</codeph>, <codeph>TINY</codeph>,
<codeph>SMALL</codeph>, <codeph>MEDIUM</codeph>.
- </li>
- </ul>
+ </p>
<p>
The terms <q>largest</q> and <q>smallest</q> refers to the size of the intermediate result set based on the
@@ -153,8 +153,7 @@ under the License.
immediately after the <codeph>SELECT</codeph> and any <codeph>DISTINCT</codeph> or <codeph>ALL</codeph>
keywords. The <codeph>STRAIGHT_JOIN</codeph> keyword turns off
the reordering of join clauses that Impala does internally, and produces a plan that relies on the join
- clauses being ordered optimally in the query text. In this case, rewrite the query so that the largest
- table is on the left, followed by the next largest, and so on until the smallest table is on the right.
+ clauses being ordered optimally in the query text.
</p>
<note>
[3/5] impala git commit: IMPALA-7090: Limit the size of expr created
by EqualityDisjunctsToInRule
Posted by tm...@apache.org.
IMPALA-7090: Limit the size of expr created by EqualityDisjunctsToInRule
Currently EqualityDisjunctsToInRule introduced in IMPALA-5280 might
create an expr with unlimited number of children and fails a query,
which should be avoided. The easy solution is to not apply the
rewrite when the number of children is large.
Change-Id: Ie40c3210271a9e3c7f1b2b869a8c2ec8bacaa72a
Reviewed-on: http://gerrit.cloudera.org:8080/10528
Reviewed-by: Tianyi Wang <tw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/aba8fdf8
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/aba8fdf8
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/aba8fdf8
Branch: refs/heads/master
Commit: aba8fdf8b2e1831986b82cec6f79fc71586e6b01
Parents: 84b55c6
Author: Tianyi Wang <ti...@apache.org>
Authored: Fri May 25 18:04:17 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed May 30 09:28:29 2018 +0000
----------------------------------------------------------------------
.../rewrite/EqualityDisjunctsToInRule.java | 32 ++++++-----
.../impala/analysis/ExprRewriterTest.java | 58 +++++++++++++++++++-
2 files changed, 75 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/aba8fdf8/fe/src/main/java/org/apache/impala/rewrite/EqualityDisjunctsToInRule.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/rewrite/EqualityDisjunctsToInRule.java b/fe/src/main/java/org/apache/impala/rewrite/EqualityDisjunctsToInRule.java
index d1a2ca7..deebefa 100644
--- a/fe/src/main/java/org/apache/impala/rewrite/EqualityDisjunctsToInRule.java
+++ b/fe/src/main/java/org/apache/impala/rewrite/EqualityDisjunctsToInRule.java
@@ -53,10 +53,10 @@ public class EqualityDisjunctsToInRule implements ExprRewriteRule {
}
/**
- * Takes the children of an OR predicate and attempts to combine them into a single IN predicate.
- * The transformation is applied if one of the children is an IN predicate and the other child
- * is a compatible IN predicate or equality predicate. Returns the transformed expr or null
- * if no transformation was possible.
+ * Takes the children of an OR predicate and attempts to combine them into a single IN
+ * predicate. The transformation is applied if one of the children is an IN predicate
+ * and the other child is a compatible IN predicate or equality predicate. Returns the
+ * transformed expr or null if no transformation was possible.
*/
private Expr rewriteInAndOtherExpr(Expr child0, Expr child1) {
InPredicate inPred = null;
@@ -64,27 +64,31 @@ public class EqualityDisjunctsToInRule implements ExprRewriteRule {
if (child0 instanceof InPredicate) {
inPred = (InPredicate) child0;
otherPred = child1;
- }
- else if (child1 instanceof InPredicate) {
+ } else if (child1 instanceof InPredicate) {
inPred = (InPredicate) child1;
otherPred = child0;
}
- if (inPred == null || inPred.isNotIn() || inPred.contains(Subquery.class)) return null;
- if (!inPred.getChild(0).equals(otherPred.getChild(0))) return null;
+ if (inPred == null || inPred.isNotIn() || inPred.contains(Subquery.class) ||
+ !inPred.getChild(0).equals(otherPred.getChild(0))) {
+ return null;
+ }
// other predicate can be OR predicate or IN predicate
List<Expr> newInList = Lists.newArrayList(
inPred.getChildren().subList(1, inPred.getChildren().size()));
if (Expr.IS_EXPR_EQ_LITERAL_PREDICATE.apply(otherPred)) {
+ if (newInList.size() + 1 == Expr.EXPR_CHILDREN_LIMIT) return null;
newInList.add(otherPred.getChild(1));
- } else
- if (otherPred instanceof InPredicate && !((InPredicate) otherPred).isNotIn()
- && !otherPred.contains(Subquery.class)) {
- newInList.addAll(
- otherPred.getChildren().subList(1, otherPred.getChildren().size()));
- } else {
+ } else if (otherPred instanceof InPredicate && !((InPredicate) otherPred).isNotIn()
+ && !otherPred.contains(Subquery.class)) {
+ if (newInList.size() + otherPred.getChildren().size() > Expr.EXPR_CHILDREN_LIMIT) {
return null;
}
+ newInList.addAll(
+ otherPred.getChildren().subList(1, otherPred.getChildren().size()));
+ } else {
+ return null;
+ }
return new InPredicate(inPred.getChild(0), newInList, false);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/aba8fdf8/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
index bebc6ba..de58d31 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
@@ -21,6 +21,7 @@ import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.rewrite.EqualityDisjunctsToInRule;
import org.apache.impala.rewrite.ExprRewriteRule;
import org.apache.impala.rewrite.ExprRewriter;
import org.junit.Assert;
@@ -30,7 +31,7 @@ import com.google.common.base.Preconditions;
/**
* Tests that the ExprRewriter framework covers all clauses as well as nested statements.
- * Does not test specific rewrite rules.
+ * It also tests some specific rewrite rules.
*/
public class ExprRewriterTest extends AnalyzerTest {
@@ -167,4 +168,59 @@ public class ExprRewriterTest extends AnalyzerTest {
RewritesOk("delete functional_kudu.testtbl where exists (" + stmt_ + ")", 24, 12);
}
}
+
+ /**
+ * construct an in-list: string_col in [offset ... offset + length)
+ */
+ private void CreateInList(int offset, int length, StringBuilder stmtSb) {
+ stmtSb.append("string_col in(");
+ for (int j = 0; j < length - 1; ++j) {
+ stmtSb.append("'c").append(offset + j).append("',");
+ }
+ stmtSb.append("'c").append(offset + length - 1).append("')");
+ }
+
+ private void CheckNumChangesByEqualityDisjunctsToInRule(
+ String stmt, int expectedNumChanges) throws ImpalaException {
+ StatementBase parsedStmt = (StatementBase) ParsesOk(stmt);
+ AnalyzesOkNoRewrite(parsedStmt);
+ ExprRewriter rewriter = new ExprRewriter(EqualityDisjunctsToInRule.INSTANCE);
+ parsedStmt.rewriteExprs(rewriter);
+ Assert.assertEquals(expectedNumChanges, rewriter.getNumChanges());
+ }
+
+ @Test
+ public void TestEqualityDisjunctsToInRuleSizeLimit() throws ImpalaException {
+ String stmtPrefix = "select count(*) from functional.alltypes where ( ";
+ // Test that EqualityDisjunctsToInRule doesn't create an expr with a number of
+ // children exceeding the limit.
+ {
+ // Create a disjunct with 2 in-lists of length Expr.EXPR_CHILDREN_LIMIT - 1.
+ StringBuilder stmtSb = new StringBuilder(stmtPrefix);
+ for (int i = 0; i < 2; ++i) {
+ int offset = (Expr.EXPR_CHILDREN_LIMIT - 1) * i;
+ CreateInList(offset, Expr.EXPR_CHILDREN_LIMIT - 1, stmtSb);
+ if (i != 1) stmtSb.append(" or ");
+ }
+ stmtSb.append(")");
+ CheckNumChangesByEqualityDisjunctsToInRule(stmtSb.toString(), 0);
+ }
+ {
+ // Create a disjunct with an in-list of length Expr.EXPR_CHILDREN_LIMIT - 1 and a
+ // EQ predicate.
+ StringBuilder stmtSb = new StringBuilder(stmtPrefix);
+ CreateInList(0, Expr.EXPR_CHILDREN_LIMIT - 1, stmtSb);
+ stmtSb.append("or string_col='").append(Expr.EXPR_CHILDREN_LIMIT - 1).append("')");
+ CheckNumChangesByEqualityDisjunctsToInRule(stmtSb.toString(), 0);
+ }
+ {
+ // Create a disjunct with an in-list of length Expr.EXPR_CHILDREN_LIMIT - 2 and 2
+ // EQ predicates.
+ StringBuilder stmtSb = new StringBuilder(stmtPrefix);
+ CreateInList(0, Expr.EXPR_CHILDREN_LIMIT - 2, stmtSb);
+ stmtSb.append("or string_col='").append(Expr.EXPR_CHILDREN_LIMIT - 2)
+ .append("' or string_col='").append(Expr.EXPR_CHILDREN_LIMIT - 1).append("')");
+ CheckNumChangesByEqualityDisjunctsToInRule(stmtSb.toString(), 1);
+ }
+ }
}
[2/5] impala git commit: Revert IMPALA-2751: Matching quotes are not
requirerd in comments
Posted by tm...@apache.org.
Revert IMPALA-2751: Matching quotes are not requirerd in comments
This patch is causing a large number of builds to fail, see
IMPALA-7089.
Change-Id: Id9995a91408d86a5ae1ecd70d07b02622ae26b43
Reviewed-on: http://gerrit.cloudera.org:8080/10537
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/84b55c61
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/84b55c61
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/84b55c61
Branch: refs/heads/master
Commit: 84b55c6148dd4e756359717cca30a8eea71cb62a
Parents: ba7893c
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Tue May 29 17:30:23 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed May 30 08:12:58 2018 +0000
----------------------------------------------------------------------
shell/impala_shell.py | 12 +++++-------
tests/shell/test_shell_interactive.py | 18 ------------------
2 files changed, 5 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/84b55c61/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 0f14d97..55ea692 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -401,8 +401,7 @@ class ImpalaShell(object, cmd.Cmd):
try:
# Look for an open quotation in the entire command, and not just the
# current line.
- if self.partial_cmd:
- line = sqlparse.format('%s %s' % (self.partial_cmd, line), strip_comments=True)
+ if self.partial_cmd: line = '%s %s' % (self.partial_cmd, line)
self._shlex_split(line)
return True
# If the command ends with a delimiter, check if it has an open quotation.
@@ -420,8 +419,8 @@ class ImpalaShell(object, cmd.Cmd):
# Iterate through the line and switch the state if a single or double quote is found
# and ignore escaped single and double quotes if the line is considered open (meaning
# a previous single or double quote has not been closed yet)
- state_closed = True
- opener = None
+ state_closed = True;
+ opener = None;
for i, char in enumerate(line):
if state_closed and (char in ['\'', '\"']):
state_closed = False
@@ -429,7 +428,7 @@ class ImpalaShell(object, cmd.Cmd):
elif not state_closed and opener == char:
if line[i - 1] != '\\':
state_closed = True
- opener = None
+ opener = None;
return state_closed
@@ -1133,8 +1132,7 @@ class ImpalaShell(object, cmd.Cmd):
query = self._create_beeswax_query(args)
# Set posix=True and add "'" to escaped quotes
# to deal with escaped quotes in string literals
- lexer = shlex.shlex(sqlparse.format(query.query.lstrip(), strip_comments=True),
- posix=True)
+ lexer = shlex.shlex(query.query.lstrip(), posix=True)
lexer.escapedquotes += "'"
# Because the WITH clause may precede DML or SELECT queries,
# just checking the first token is insufficient.
http://git-wip-us.apache.org/repos/asf/impala/blob/84b55c61/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 2f5278f..f53ee5a 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -516,24 +516,6 @@ class TestImpalaShellInteractive(object):
assert '| 1"23"4 |' in result.stdout
@pytest.mark.execute_serially
- def test_comment_with_quotes(self):
- # IMPALA-2751: Comment does not need to have matching quotes
- queries = [
- "select -- '\n1;",
- 'select -- "\n1;',
- "select -- \"'\n 1;",
- "select /*'\n*/ 1;",
- 'select /*"\n*/ 1;',
- "select /*\"'\n*/ 1;",
- "with a as (\nselect 1\n-- '\n) select * from a",
- 'with a as (\nselect 1\n-- "\n) select * from a',
- "with a as (\nselect 1\n-- '\"\n) select * from a",
- ]
- for query in queries:
- result = run_impala_shell_interactive(query)
- assert '| 1 |' in result.stdout
-
- @pytest.mark.execute_serially
def test_shell_prompt(self):
proc = pexpect.spawn(SHELL_CMD)
proc.expect(":21000] default>")