You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/08/21 15:25:12 UTC
[hive] branch branch-3 updated: HIVE-21980: Parsing time can be
high in case of deeply nested subqueries (Zoltan Haindrich reviewed by
Vineet Garg)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new dcfbf18 HIVE-21980: Parsing time can be high in case of deeply nested subqueries (Zoltan Haindrich reviewed by Vineet Garg)
dcfbf18 is described below
commit dcfbf18cc2aa8f66c6eb7f78c7f041f7031382d2
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Tue Jul 16 10:50:09 2019 +0200
HIVE-21980: Parsing time can be high in case of deeply nested subqueries (Zoltan Haindrich reviewed by Vineet Garg)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
(cherry picked from commit 0f39030c3d33b11ae9c14ac81e047b44e8695371)
---
.../apache/hadoop/hive/ql/parse/FromClauseParser.g | 16 ++--
.../hadoop/hive/ql/parse/TestParseDriver.java | 94 +++++++++++++++++++++-
.../test/results/clientnegative/subq_insert.q.out | 2 +-
3 files changed, 99 insertions(+), 13 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
index e2309af..e6b6fd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
@@ -96,16 +96,12 @@ fromSource
atomjoinSource
@init { gParent.pushMsg("joinSource", state); }
@after { gParent.popMsg(state); }
- :
- tableSource (lateralView^)*
- |
- virtualTableSource (lateralView^)*
- |
- (subQuerySource) => subQuerySource (lateralView^)*
- |
- partitionedTableFunction (lateralView^)*
- |
- LPAREN! joinSource RPAREN!
+ : tableSource (lateralView^)*
+ | virtualTableSource (lateralView^)*
+ | (LPAREN (KW_WITH|KW_SELECT|KW_MAP|KW_REDUCE|KW_FROM)) => subQuerySource (lateralView^)*
+ | (LPAREN LPAREN atomSelectStatement RPAREN setOperator ) => subQuerySource (lateralView^)*
+ | partitionedTableFunction (lateralView^)*
+ | LPAREN! joinSource RPAREN!
;
joinSource
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java
index c5d099d..f3372af 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java
@@ -19,10 +19,14 @@ package org.apache.hadoop.hive.ql.parse;
import static org.junit.Assert.assertEquals;
+import java.io.File;
+import java.nio.charset.Charset;
+
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
+import com.google.common.io.Files;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class TestParseDriver {
@@ -40,7 +44,7 @@ public class TestParseDriver {
String whereStr = "field5=1 and field6 in ('a', 'b')";
String havingStr = "sum(field7) > 11";
ASTNode tree = parseDriver.parse(selectStr + " from table1 where " + whereStr
- + " group by field1, field2 having " + havingStr);
+ + " group by field1, field2 having " + havingStr);
assertEquals(tree.getType(), 0);
assertEquals(tree.getChildCount(), 2);
ASTNode queryTree = (ASTNode) tree.getChild(0);
@@ -106,7 +110,7 @@ public class TestParseDriver {
assertTree((ASTNode) sumNode.getChild(1), plusNode);
ASTNode tree = parseDriver.parseExpression("case when field1 = 1 then sum(field3 + field4) when field1 != 2 then " +
- "sum(field3-field4) else sum(field3 * field4) end");
+ "sum(field3-field4) else sum(field3 * field4) end");
assertEquals(tree.getChildCount(), 6);
assertEquals(tree.getChild(0).getType(), HiveParser.KW_WHEN);
assertEquals(tree.getChild(1).getType(), HiveParser.EQUAL);
@@ -214,4 +218,90 @@ public class TestParseDriver {
"AS test_comp_exp");
}
+ static class ExoticQueryBuilder {
+ StringBuilder sb = new StringBuilder();
+
+ public void recursiveSJS(int depth) {
+ sb.append("select ");
+ addColumns(30);
+ sb.append(" from \n");
+ tablePart(depth);
+ sb.append(" join \n");
+ tablePart(depth);
+ sb.append(" on ( ");
+ wherePart(10);
+ sb.append(" ) ");
+ sb.append(" where ");
+ wherePart(10);
+
+ }
+
+ private void tablePart(int depth) {
+ if (depth == 0) {
+ sb.append(" baseTable ");
+ } else {
+ sb.append("(");
+ recursiveSJS(depth - 1);
+ sb.append(") aa");
+ }
+ }
+
+ private void wherePart(int num) {
+ for (int i = 0; i < num - 1; i++) {
+ sb.append("x = ");
+ sb.append(i);
+ sb.append(" or ");
+ }
+ sb.append("x = -1");
+
+ }
+
+ private void addColumns(int num) {
+ for (int i = 0; i < num - 1; i++) {
+ sb.append("c");
+ sb.append(i);
+ sb.append(" + 2*sqrt(11)+");
+ sb.append(i);
+ sb.append(",");
+ }
+ sb.append("cE");
+ }
+
+ public String getQuery() {
+ return sb.toString();
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testExoticSJSSubQuery() throws Exception {
+ ExoticQueryBuilder eqb = new ExoticQueryBuilder();
+ eqb.recursiveSJS(10);
+ String q = eqb.getQuery();
+ System.out.println(q);
+ parseDriver.parse(q);
+ }
+
+ @Test
+ public void testJoinResulInBraces() throws Exception {
+ String q =
+ "explain select a.key, b.value from"
+ + "( (select key from src)a join (select value from src)b on a.key=b.value)";
+ System.out.println(q);
+
+ ASTNode root = parseDriver.parse(q);
+ System.out.println(root.dump());
+
+ }
+
+ @Test
+ public void testFromSubqueryIsSetop() throws Exception {
+ String q =
+ "explain select key from ((select key from src) union (select key from src))subq ";
+ System.out.println(q);
+
+ ASTNode root = parseDriver.parse(q);
+ System.out.println(root.dump());
+
+ }
+
}
\ No newline at end of file
diff --git a/ql/src/test/results/clientnegative/subq_insert.q.out b/ql/src/test/results/clientnegative/subq_insert.q.out
index 620409b..8ac6815 100644
--- a/ql/src/test/results/clientnegative/subq_insert.q.out
+++ b/ql/src/test/results/clientnegative/subq_insert.q.out
@@ -1 +1 @@
-FAILED: SemanticException [Error 10024]: Line 2:38 Cannot insert in a subquery. Inserting to table 'src1'
+FAILED: ParseException line 2:32 cannot recognize input near '(' 'INSERT' 'OVERWRITE' in joinSource