You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/13 08:45:41 UTC
svn commit: r1481706 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/metadata/
java/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/processors/
Author: amareshwari
Date: Mon May 13 06:45:40 2013
New Revision: 1481706
URL: http://svn.apache.org/r1481706
Log:
Fix alias replacer not to do string replace
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java Mon May 13 06:45:40 2013
@@ -195,10 +195,10 @@ public final class Cube extends Abstract
}
public CubeDimension getDimensionByName(String dimension) {
- return dimMap.get(dimension);
+ return dimMap.get(dimension == null ? dimension : dimension.toLowerCase());
}
public CubeMeasure getMeasureByName(String measure) {
- return measureMap.get(measure);
+ return measureMap.get(measure == null ? measure : measure.toLowerCase());
}
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java Mon May 13 06:45:40 2013
@@ -45,7 +45,7 @@ public class AggregateResolver implement
validateAggregates(cubeql, cubeql.getSelectAST(), false, false, false);
validateAggregates(cubeql, cubeql.getHavingAST(), false, false, false);
-
+ System.out.println("Before aggregate resolver: " + cubeql.getSelectTree());
String rewritSelect = resolveForSelect(cubeql, cubeql.getSelectTree());
System.out.println("New select after aggregate resolver: " + rewritSelect);
cubeql.setSelectTree(rewritSelect);
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java Mon May 13 06:45:40 2013
@@ -2,7 +2,12 @@ package org.apache.hadoop.hive.ql.cube.p
import java.util.Map;
+import org.antlr.runtime.CommonToken;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
public class AliasReplacer implements ContextRewriter {
@@ -10,14 +15,14 @@ public class AliasReplacer implements Co
}
@Override
- public void rewriteContext(CubeQueryContext cubeql) {
+ public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
// Rewrite the all the columns in the query with table alias prefixed.
// If col1 of table tab1 is accessed, it would be changed as tab1.col1.
// If tab1 is already aliased say with t1, col1 is changed as t1.col1
// replace the columns in select, groupby, having, orderby by
// prepending the table alias to the col
- // sample select trees
+ //sample select trees
// 1: (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))
// (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value))))
// 2: (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key))
@@ -27,40 +32,90 @@ public class AliasReplacer implements Co
if (colToTableAlias == null) {
return;
}
- String selectTree = cubeql.getSelectTree();
- String whereTree = cubeql.getWhereTree();
- String havingTree = cubeql.getHavingTree();
- String orderByTree = cubeql.getOrderByTree();
- String groupByTree = cubeql.getGroupByTree();
- if (selectTree != null) {
- cubeql.setSelectTree(replaceCols(selectTree, colToTableAlias));
- }
-
- if (whereTree != null) {
- cubeql.setWhereTree(replaceCols(whereTree, colToTableAlias));
- }
-
- if (havingTree != null) {
- cubeql.setHavingTree(replaceCols(havingTree, colToTableAlias));
+ ASTNode selectAST = cubeql.getSelectAST();
+ replaceAliases(selectAST, 0, colToTableAlias);
+ String rewritSelect = HQLParser.getString(selectAST);
+ if (StringUtils.isNotBlank(rewritSelect)) {
+ cubeql.setSelectTree(rewritSelect);
+ }
+
+ ASTNode havingAST = cubeql.getHavingAST();
+ replaceAliases(havingAST, 0, colToTableAlias);
+ String rewritHaving = HQLParser.getString(havingAST);
+ if (StringUtils.isNotBlank(rewritHaving)) {
+ cubeql.setHavingTree(rewritHaving);
+ }
+
+ ASTNode orderByAST = cubeql.getOrderByAST();
+ replaceAliases(orderByAST, 0, colToTableAlias);
+ String rewritOrderby = HQLParser.getString(orderByAST);
+ if (StringUtils.isNotBlank(rewritOrderby)) {
+ cubeql.setOrderByTree(rewritOrderby);
+ }
+
+ ASTNode groupByAST = cubeql.getGroupByAST();
+ replaceAliases(groupByAST, 0, colToTableAlias);
+ String rewritGroupby = HQLParser.getString(groupByAST);
+ if (StringUtils.isNotBlank(rewritGroupby)) {
+ cubeql.setGroupByTree(rewritGroupby);
+ }
+
+ ASTNode whereAST = cubeql.getWhereAST();
+ replaceAliases(whereAST, 0, colToTableAlias);
+ String rewritWhere = HQLParser.getString(whereAST);
+ if (StringUtils.isNotBlank(rewritWhere)) {
+ cubeql.setWhereTree(rewritWhere);
}
+ }
- if (orderByTree != null) {
- cubeql.setOrderByTree(replaceCols(orderByTree, colToTableAlias));
+ private void replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) {
+ if (node == null) {
+ return;
}
- if (groupByTree != null) {
- cubeql.setGroupByTree(replaceCols(groupByTree, colToTableAlias));
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+ String colName = HQLParser.getColName(node);
+ String newAlias = colToTableAlias.get(colName.toLowerCase());
+
+ if (StringUtils.isBlank(newAlias)) {
+ return;
+ }
+
+ if (nodeType == HiveParser.DOT) {
+ // No need to create a new node, just replace the table name ident
+ ASTNode aliasNode = (ASTNode) node.getChild(0);
+ ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
+ aliasNode.setChild(0, newAliasIdent);
+ newAliasIdent.setParent(aliasNode);
+ } else {
+ // Just a column ref, we need to make it alias.col
+ // '.' will become the parent node
+ ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, "."));
+ ASTNode aliasIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
+ ASTNode tabRefNode = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"));
+
+ tabRefNode.addChild(aliasIdentNode);
+ aliasIdentNode.setParent(tabRefNode);
+ dot.addChild(tabRefNode);
+ tabRefNode.setParent(dot);
+
+ ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName));
+
+ dot.addChild(colIdentNode);
+
+ ASTNode parent = (ASTNode) node.getParent();
+
+ parent.setChild(nodePos, dot);
+ }
+ } else {
+ // recurse down
+ for (int i = 0; i < node.getChildCount(); i++) {
+ ASTNode child = (ASTNode) node.getChild(i);
+ replaceAliases(child, i, colToTableAlias);
+ }
}
-
}
- private String replaceCols(String tree, Map<String, String> colToTableAlias) {
- String srcTree = new String(tree);
- for (Map.Entry<String, String> entry : colToTableAlias.entrySet()) {
- srcTree = srcTree.replaceAll(entry.getKey(),
- entry.getValue() + "." + entry.getKey());
- }
- return srcTree;
- }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Mon May 13 06:45:40 2013
@@ -111,6 +111,9 @@ public class CubeQueryContext {
private final ASTNode joinTree;
private ASTNode havingAST;
private ASTNode selectAST;
+ private ASTNode whereAST;
+ private ASTNode orderByAST;
+ private ASTNode groupByAST;
public CubeQueryContext(ASTNode ast, QB qb, HiveConf conf)
throws SemanticException {
@@ -121,20 +124,23 @@ public class CubeQueryContext {
if (qb.getParseInfo().getWhrForClause(clauseName) != null) {
this.whereTree = HQLParser.getString(
qb.getParseInfo().getWhrForClause(clauseName)).toLowerCase();
+ this.whereAST = qb.getParseInfo().getWhrForClause(clauseName);
}
if (qb.getParseInfo().getHavingForClause(clauseName) != null) {
- this.havingTree = HQLParser.getString(qb.getParseInfo()
- .getHavingForClause(clauseName)).toLowerCase();
+ this.havingTree = HQLParser.getString(qb.getParseInfo().getHavingForClause(
+ clauseName)).toLowerCase();
this.havingAST = qb.getParseInfo().getHavingForClause(
clauseName);
}
if (qb.getParseInfo().getOrderByForClause(clauseName) != null) {
this.orderByTree = HQLParser.getString(qb.getParseInfo()
.getOrderByForClause(clauseName)).toLowerCase();
+ this.orderByAST = qb.getParseInfo().getOrderByForClause(clauseName);
}
if (qb.getParseInfo().getGroupByForClause(clauseName) != null) {
this.groupByTree = HQLParser.getString(qb.getParseInfo()
.getGroupByForClause(clauseName)).toLowerCase();
+ this.groupByAST = qb.getParseInfo().getGroupByForClause(clauseName);
}
if (qb.getParseInfo().getSelForClause(clauseName) != null) {
this.selectTree = HQLParser.getString(qb.getParseInfo().getSelForClause(
@@ -977,21 +983,25 @@ public class CubeQueryContext {
}
public boolean isCubeMeasure(String col) {
+ if (col == null) {
+ return false;
+ }
+
+ col = col.trim();
// Take care of brackets added around col names in HQLParsrer.getString
if (col.startsWith("(") && col.endsWith(")") && col.length() > 2) {
- col = col.substring(1, col.length() - 1);
+ col = col.substring(1, col.length() -1);
}
String[] split = StringUtils.split(col, ".");
if (split.length <= 1) {
- return cubeMeasureNames.contains(col);
+ return cubeMeasureNames.contains(col.trim().toLowerCase());
} else {
String cubeName = split[0].trim();
String colName = split[1].trim();
if (cubeName.equalsIgnoreCase(cube.getName()) ||
cubeName.equalsIgnoreCase(getAliasForTabName(cube.getName()))) {
- boolean ismeasure = cubeMeasureNames.contains(colName);
- return cubeMeasureNames.contains(colName);
+ return cubeMeasureNames.contains(colName.toLowerCase());
} else {
return false;
}
@@ -1007,9 +1017,8 @@ public class CubeQueryContext {
}
public boolean hasAggregates() {
- return !aggregateExprs.isEmpty() || (cube != null);
+ return !aggregateExprs.isEmpty() || (cube !=null);
}
-
public String getAlias(String expr) {
return exprToAlias.get(expr);
}
@@ -1046,4 +1055,17 @@ public class CubeQueryContext {
joinConds.put(qb, cond);
}
+ public ASTNode getWhereAST() {
+ return whereAST;
+ }
+
+ public ASTNode getOrderByAST() {
+ return orderByAST;
+ }
+
+ public ASTNode getGroupByAST() {
+ return groupByAST;
+ }
+
+
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java Mon May 13 06:45:40 2013
@@ -25,11 +25,11 @@ public class CubeQueryRewriter {
private void setupPhase1Rewriters() {
// Resolve joins and generate base join tree
phase1Rewriters.add(new JoinResolver(conf));
+ // Rewrite base trees (groupby, having, orderby, limit) using aliases
+ phase1Rewriters.add(new AliasReplacer(conf));
// Resolve aggregations and generate base select tree
phase1Rewriters.add(new AggregateResolver(conf));
phase1Rewriters.add(new GroupbyResolver(conf));
- // Rewrite base trees (groupby, having, orderby, limit) using aliases
- phase1Rewriters.add(new AliasReplacer(conf));
// Resolve partition columns and table names
phase1Rewriters.add(new PartitionResolver(conf));
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java Mon May 13 06:45:40 2013
@@ -301,4 +301,18 @@ public class HQLParser {
toInfixString(tree, buf);
return buf.toString();
}
-}
+
+ public static String getColName(ASTNode node) {
+ String colname = null;
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
+ colname = ((ASTNode) node.getChild(0)).getText();
+ } else {
+ // node in 'alias.column' format
+ ASTNode colIdent = (ASTNode) node.getChild(1);
+ colname = colIdent.getText();
+ }
+
+ return colname;
+ }
+}
\ No newline at end of file
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1481706&r1=1481705&r2=1481706&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Mon May 13 06:45:40 2013
@@ -335,7 +335,7 @@ public class TestCubeDriver {
// Should throw exception in aggregate resolver because noAggrMsr does
//not have a default aggregate defined.s
String hql = driver.compileCubeQuery(q8);
- Assert.assertTrue("Should not reach here", false);
+ Assert.assertTrue("Should not reach here: " + hql, false);
} catch (SemanticException exc) {
Assert.assertNotNull(exc);
exc.printStackTrace();
@@ -349,7 +349,7 @@ public class TestCubeDriver {
try {
String hql = driver.compileCubeQuery(query);
- Assert.assertTrue("Should not reach here", false);
+ Assert.assertTrue("Should not reach here:" + hql, false);
} catch (SemanticException exc) {
Assert.assertNotNull(exc);
exc.printStackTrace();
@@ -358,7 +358,7 @@ public class TestCubeDriver {
String q2 = "SELECT ambigdim2, testCube.msr1 FROM testCube " + timeRange;
try {
String hql = driver.compileCubeQuery(q2);
- Assert.assertTrue("Should not reach here", false);
+ Assert.assertTrue("Should not reach here: " + hql, false);
} catch (SemanticException exc) {
Assert.assertNotNull(exc);
exc.printStackTrace();
@@ -418,4 +418,26 @@ public class TestCubeDriver {
}
}
+ @Test
+ public void testAliasReplacer() throws Exception {
+ String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
+
+ System.out.println("##TEST_ALIAS_REPLACER");
+ String queries[] = {
+ "SELECT cityid, t.msr2 FROM testCube t" + timeRange,
+ "SELECT cityid, msr2 FROM testCube " + timeRange + " and msr2 > 100 HAVING msr2 < 1000",
+ "SELECT cityid, testCube.msr2 FROM tetCube " + timeRange + " and msr2 > 100 HAVING msr2 < 1000 ORDER BY cityid"
+ };
+
+ try {
+ for (String q : queries) {
+ String hql = driver.compileCubeQuery(q);
+ System.out.println("@@QUERY: " + q);
+ System.out.println("@@HQL: " + hql);
+ }
+ } catch (Exception exc) {
+ exc.printStackTrace();
+ }
+ }
+
}