You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/06 13:10:40 UTC
svn commit: r1479522 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/metadata/
java/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/processors/
Author: amareshwari
Date: Mon May 6 11:10:40 2013
New Revision: 1479522
URL: http://svn.apache.org/r1479522
Log:
Add aggregate resolver
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java Mon May 6 11:10:40 2013
@@ -16,6 +16,10 @@ public final class Cube extends Abstract
private final Set<CubeMeasure> measures;
private final Set<CubeDimension> dimensions;
private static final List<FieldSchema> columns = new ArrayList<FieldSchema>();
+ private final Map<String, CubeMeasure> measureMap;
+ private final Map<String, CubeDimension> dimMap;
+
+
static {
columns.add(new FieldSchema("dummy", "string", "dummy column"));
}
@@ -25,6 +29,17 @@ public final class Cube extends Abstract
super(name, columns, new HashMap<String, String>());
this.measures = measures;
this.dimensions = dimensions;
+
+ measureMap = new HashMap<String, CubeMeasure>();
+ for (CubeMeasure m : measures) {
+ measureMap.put(m.getName(), m);
+ }
+
+ dimMap = new HashMap<String, CubeDimension>();
+ for (CubeDimension dim : dimensions) {
+ dimMap.put(dim.getName(), dim);
+ }
+
addProperties();
}
@@ -32,6 +47,15 @@ public final class Cube extends Abstract
super(tbl);
this.measures = getMeasures(getName(), getProperties());
this.dimensions = getDimensions(getName(), getProperties());
+ measureMap = new HashMap<String, CubeMeasure>();
+ for (CubeMeasure m : measures) {
+ measureMap.put(m.getName(), m);
+ }
+
+ dimMap = new HashMap<String, CubeDimension>();
+ for (CubeDimension dim : dimensions) {
+ dimMap.put(dim.getName(), dim);
+ }
}
public Set<CubeMeasure> getMeasures() {
@@ -169,4 +193,12 @@ public final class Cube extends Abstract
}
return true;
}
+
+ public CubeDimension getDimensionByName(String dimension) {
+ return dimMap.get(dimension);
+ }
+
+ public CubeMeasure getMeasureByName(String measure) {
+ return measureMap.get(measure);
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java Mon May 6 11:10:40 2013
@@ -1,18 +1,322 @@
package org.apache.hadoop.hive.ql.cube.parse;
-import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
+
+import java.util.Arrays;
+import org.antlr.runtime.CommonToken;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeMeasure;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.log4j.Logger;
+/**
+ * <p>Replace select and having columns with default aggregate functions on
+ * them, if default aggregate is defined and if there isn't already an
+ * aggregate function specified on the columns.</p>
+ *
+ * <p>For example, if query is like - <pre>select dim1.name, fact.msr1, fact.msr1 * sin(fact.msr2)/cos(fact.msr2),
+ * sum(fact.msr4) ...</pre>
+ * Then we will replace fact.msr1 with sum(fact.msr1) given that 'sum' has been
+ * set as the default aggregate function for msr1.</p>
+ *
+ * <p>We will also wrap expressions of measures into a default aggregate function.
+ * For example the expression 'fact.msr1 * sin(fact.msr2)/cos(fact.msr2)' will become
+ * sum(fact.msr1 * sin(fact.msr2)/cos(fact.msr2)), if sum is the default aggregate function.</p>
+ *
+ * <p>Expressions which already contain aggregate sub-expressions will not be changed.</p>
+ *
+ * <p>At this point it's assumed that aliases have been added to all columns.</p>
+ */
public class AggregateResolver implements ContextRewriter {
+ public static final Logger LOG = Logger.getLogger(AggregateResolver.class);
+
+ private final Configuration conf;
public AggregateResolver(Configuration conf) {
- // TODO Auto-generated constructor stub
+ this.conf = conf;
}
@Override
- public void rewriteContext(CubeQueryContext cubeql) {
- // TODO
- // replace select and having columns with default aggregate functions on
- // them, if default aggregate is defined
+ public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
+ if (cubeql.getCube() == null) {
+ LOG.warn("AggregateResolver called without a cube setup. Returning");
+ return;
+ }
+
+ validateAggregates(cubeql, cubeql.getSelectAST(), false, false, false);
+ validateAggregates(cubeql, cubeql.getHavingAST(), false, false, false);
+
+ String rewritSelect = resolveForSelect(cubeql, cubeql.getSelectTree());
+ System.out.println("New select after aggregate resolver: " + rewritSelect);
+ cubeql.setSelectTree(rewritSelect);
+
+ String rewritHaving = resolveForHaving(cubeql);
+ if (StringUtils.isNotBlank(rewritHaving)) {
+ cubeql.setHavingTree(rewritHaving);
+ }
+ System.out.println("New having after aggregate resolver: " + rewritHaving);
+ }
+
+ private void validateAggregates(CubeQueryContext cubeql, ASTNode node, boolean insideAggregate,
+ boolean insideArithExpr, boolean insideNonAggrFn) throws SemanticException {
+ if (node == null) {
+ return;
+ }
+
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+ // Found a column ref. If this is a measure, it should be inside an aggregate if its part of
+ // an arithmetic expression or an argument of a non-aggregate function
+ String msrname = getColName(node);
+ if (cubeql.isCubeMeasure(msrname) &&
+ !insideAggregate
+ && (insideArithExpr || insideNonAggrFn)) {
+ throw new SemanticException("Not inside aggregate " + msrname);
+ }
+ } else if (HQLParser.isArithmeticOp(nodeType)) {
+ // Allowed - sum ( msr1 * msr2 + msr3)
+ // Not allowed - msr1 + msr2 * msr3 <- Not inside aggregate
+ // Not allowed - sum(msr1) + msr2 <- Aggregate only on one measure
+ // count of measures within aggregates must be equal to count of measures
+ // if both counts are equal and zero, then this node should be inside aggregate
+ int measuresInAggregates = countMeasuresInAggregates(cubeql, node, false);
+ int measuresInTree = countMeasures(cubeql, node);
+
+ if (measuresInAggregates == measuresInTree) {
+ if (measuresInAggregates == 0 && !insideAggregate) {
+ // (msr1 + msr2)
+ throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+ } else if (insideAggregate) {
+ // sum(sum(msr1) + sum(msr2))
+ throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+ }
+ } else {
+ throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+ }
+ } else {
+ boolean isArithmetic = HQLParser.isArithmeticOp(nodeType);
+ boolean isAggregate = isAggregateAST(node);
+ boolean isNonAggrFn = nodeType == HiveParser.TOK_FUNCTION && !isAggregate;
+ for (int i = 0; i < node.getChildCount(); i++) {
+ validateAggregates(cubeql, (ASTNode) node.getChild(i), isAggregate, isArithmetic, isNonAggrFn);
+ }
+ }
+ }
+
+ private int countMeasures(CubeQueryContext cubeql, ASTNode node) {
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+ String msrname = getColName(node);
+ if (cubeql.isCubeMeasure(msrname)) {
+ return 1;
+ } else {
+ return 0;
+ }
+ } else {
+ int count = 0;
+ for (int i = 0; i < node.getChildCount(); i++) {
+ count += countMeasures(cubeql, (ASTNode) node.getChild(i));
+ }
+ return count;
+ }
+ }
+
+ private int countMeasuresInAggregates(CubeQueryContext cubeql, ASTNode node, boolean inAggregate) {
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+ String msrname = getColName(node);
+ if (cubeql.isCubeMeasure(msrname) && inAggregate) {
+ return 1;
+ } else {
+ return 0;
+ }
+ } else {
+ int count = 0;
+ for (int i = 0; i < node.getChildCount(); i++) {
+ boolean isAggr = isAggregateAST((ASTNode)node.getChild(i));
+ count += countMeasuresInAggregates(cubeql, (ASTNode) node.getChild(i), isAggr);
+ }
+ return count;
+ }
+ }
+
+ private String resolveForSelect(CubeQueryContext cubeql, String exprTree) {
+ // Aggregate resolver needs cube to be resolved first
+ assert cubeql.getCube() != null;
+
+ if (StringUtils.isBlank(exprTree)) {
+ return "";
+ }
+
+ System.out.println("resolveAggregatesForExpr - " + exprTree);
+
+ String exprTokens[] = StringUtils.split(exprTree, ",");
+ for (int i = 0; i < exprTokens.length; i++) {
+ String token = exprTokens[i].trim();
+ System.out.println("_NEW_TOKEN_[" + token + "]");
+ String tokenAlias = cubeql.getAlias(token);
+ boolean hasAlias = false;
+ if (StringUtils.isNotBlank(tokenAlias)) {
+ token = token.substring(0, exprTree.lastIndexOf(tokenAlias)).trim();
+ hasAlias = true;
+ }
+
+ if (!cubeql.isAggregateExpr(token)) {
+ if (cubeql.isCubeMeasure(token)) {
+ // Take care of brackets added around col names in HQLParsrer.getString
+ if (token.startsWith("(") && token.endsWith(")") && token.length() > 2) {
+ token = token.substring(1, token.length() -1);
+ }
+
+ String splits[] = StringUtils.split(token, ".");
+ for (int j = 0; j < splits.length; j++) {
+ splits[j] = splits[j].trim();
+ }
+
+ System.out.println(">> TOKEN:" + token + " SPLITS:" + Arrays.toString(splits));
+ String msrName = (splits.length <= 1) ? splits[0] : splits[1];
+ CubeMeasure measure = cubeql.getCube().getMeasureByName(msrName);
+ if (measure != null) {
+ String msrAggregate = measure.getAggregate();
+
+ if (StringUtils.isNotBlank(msrAggregate)) {
+ System.out.println("#replace msrName:["+msrName+"] "+ " with msrAggregate["+msrAggregate+"]");
+ exprTokens[i] = msrAggregate + "( " + token + ")" + (hasAlias ? " " + tokenAlias : "");
+ exprTokens[i] = exprTokens[i].toLowerCase();
+ // Add this expression to aggregate expr set so that group by resolver can skip
+ // over expressions changed during aggregate resolver.
+ cubeql.addAggregateExpr(exprTokens[i]);
+ } else {
+ LOG.warn("Default aggregate not specified. measure:" + msrName + " token:" + token);
+ }
+ }
+ }
+ } else {
+ System.out.println("Aggregate already specified: " + token);
+ }
+ }
+
+ return StringUtils.join(exprTokens, ", ");
}
+ // We need to traverse the AST for Having clause.
+ // We need to skip any columns that are inside an aggregate UDAF or inside an arithmetic expression
+ private String resolveForHaving(CubeQueryContext cubeql) {
+ ASTNode havingTree = cubeql.getHavingAST();
+ String havingTreeStr = cubeql.getHavingTree();
+
+ if (StringUtils.isBlank(havingTreeStr) || havingTree == null) {
+ return null;
+ }
+
+ for (int i = 0; i < havingTree.getChildCount(); i++) {
+ transform(cubeql, havingTree, (ASTNode) havingTree.getChild(i), i);
+ }
+
+ return HQLParser.getString(havingTree);
+ }
+
+ private void transform(CubeQueryContext cubeql, ASTNode parent, ASTNode node, int nodePos) {
+ if (parent == null || node == null) {
+ return;
+ }
+ int nodeType = node.getToken().getType();
+
+ if (! (isAggregateAST(node) || HQLParser.isArithmeticOp(nodeType))) {
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+ // Leaf node
+ ASTNode wrapped = wrapAggregate(cubeql, node);
+ if (wrapped != node) {
+ parent.setChild(nodePos, wrapped);
+ }
+ } else {
+ // Dig deeper in non-leaf nodes
+ for (int i = 0; i < node.getChildCount(); i++) {
+ transform(cubeql, node, (ASTNode) node.getChild(i), i);
+ }
+ }
+ }
+ }
+
+ private boolean isAggregateAST(ASTNode node) {
+ int exprTokenType = node.getToken().getType();
+ if (exprTokenType == HiveParser.TOK_FUNCTION
+ || exprTokenType == HiveParser.TOK_FUNCTIONDI
+ || exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
+ assert (node.getChildCount() != 0);
+ if (node.getChild(0).getType() == HiveParser.Identifier) {
+ String functionName = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0)
+ .getText());
+ if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ // Wrap an aggregate function around the node if its a measure, leave it unchanged otherwise
+ private ASTNode wrapAggregate(CubeQueryContext cubeql, ASTNode node) {
+
+ String tabname = null;
+ String colname = null;
+
+ if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
+ colname = ((ASTNode) node.getChild(0)).getText();
+ } else {
+ // node in 'alias.column' format
+ ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
+ Identifier);
+ ASTNode colIdent = (ASTNode) node.getChild(1);
+
+ colname = colIdent.getText();
+ tabname = tabident.getText();
+ }
+
+ String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
+
+ if (cubeql.isCubeMeasure(msrname)) {
+ CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
+ String aggregateFn = measure.getAggregate();
+ ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
+ fnroot.setParent(node.getParent());
+
+ ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn));
+ fnIdentNode.setParent(fnroot);
+ fnroot.addChild(fnIdentNode);
+
+ node.setParent(fnroot);
+ fnroot.addChild(node);
+
+ return fnroot;
+ } else {
+ return node;
+ }
+ }
+
+ private String getColName(ASTNode node) {
+ String tabname = null;
+ String colname = null;
+ int nodeType = node.getToken().getType();
+ if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
+ colname = ((ASTNode) node.getChild(0)).getText();
+ } else {
+ // node in 'alias.column' format
+ ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
+ Identifier);
+ ASTNode colIdent = (ASTNode) node.getChild(1);
+
+ colname = colIdent.getText();
+ tabname = tabident.getText();
+ }
+
+ return StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Mon May 6 11:10:40 2013
@@ -19,6 +19,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.cube.metadata.AbstractCubeTable;
import org.apache.hadoop.hive.ql.cube.metadata.Cube;
@@ -95,6 +96,8 @@ public class CubeQueryContext {
private String groupByTree;
private ASTNode limitTree;
private final ASTNode joinTree;
+ private ASTNode havingAST;
+ private ASTNode selectAST;
public CubeQueryContext(ASTNode ast, QB qb, HiveConf conf)
throws SemanticException {
@@ -109,6 +112,8 @@ public class CubeQueryContext {
if (qb.getParseInfo().getHavingForClause(clauseName) != null) {
this.havingTree = HQLParser.getString(qb.getParseInfo().getHavingForClause(
clauseName)).toLowerCase();
+ this.havingAST = qb.getParseInfo().getHavingForClause(
+ clauseName);
}
if (qb.getParseInfo().getOrderByForClause(clauseName) != null) {
this.orderByTree = HQLParser.getString(qb.getParseInfo()
@@ -121,6 +126,8 @@ public class CubeQueryContext {
if (qb.getParseInfo().getSelForClause(clauseName) != null) {
this.selectTree = HQLParser.getString(qb.getParseInfo().getSelForClause(
clauseName)).toLowerCase();
+ this.selectAST = qb.getParseInfo().getSelForClause(
+ clauseName);
}
this.joinTree = qb.getParseInfo().getJoinExpr();
extractMetaTables();
@@ -550,9 +557,9 @@ public class CubeQueryContext {
if (joinTree.getJoinCond() != null) {
builder.append("\n joinConds:");
for (JoinCond cond: joinTree.getJoinCond()) {
- builder.append("\n\t left: " + cond.getLeft() + " right: "
- + cond.getRight() + " type:" + cond.getJoinType()
- + " preserved:" + cond.getPreserved());
+ builder.append("\n\t left: " + cond.getLeft() + " right: " +
+ cond.getRight() + " type:" + cond.getJoinType() +
+ " preserved:" + cond.getPreserved());
}
}
if (joinTree.getExpressions() != null) {
@@ -759,7 +766,7 @@ public class CubeQueryContext {
factStorageTable));
} else {
if (originalWhereString != null) {
- whereWithoutTimerange = new StringBuilder(originalWhereString);
+ whereWithoutTimerange = new StringBuilder(originalWhereString);
} else {
whereWithoutTimerange = new StringBuilder();
}
@@ -885,13 +892,23 @@ public class CubeQueryContext {
}
public boolean isCubeMeasure(String col) {
- String[] split = col.split("\\.");
+ // Take care of brackets added around col names in HQLParsrer.getString
+ if (col.startsWith("(") && col.endsWith(")") && col.length() > 2) {
+ col = col.substring(1, col.length() -1);
+ }
+
+ String[] split = StringUtils.split(col, ".");
+ System.out.println("Looking for col [" + col + "] split: " + split.length);
if (split.length <= 1) {
return cubeMeasureNames.contains(col);
} else {
- if (split[0].equalsIgnoreCase(cube.getName()) ||
- split[0].equalsIgnoreCase(getAliasForTabName(cube.getName()))) {
- return cubeMeasureNames.contains(split[1]);
+ String cubeName = split[0].trim();
+ String colName = split[1].trim();
+ if (cubeName.equalsIgnoreCase(cube.getName()) ||
+ cubeName.equalsIgnoreCase(getAliasForTabName(cube.getName()))) {
+ boolean ismeasure = cubeMeasureNames.contains(colName);
+ System.out.println(colName + " IS MEASURE? " + ismeasure);
+ return cubeMeasureNames.contains(colName);
} else {
return false;
}
@@ -925,4 +942,16 @@ public class CubeQueryContext {
candidateFactTables.remove(fact);
}
+ public void addAggregateExpr(String expr) {
+ aggregateExprs.add(expr);
+ }
+
+ public ASTNode getHavingAST() {
+ return havingAST;
+ }
+
+ public ASTNode getSelectAST() {
+ return selectAST;
+ }
+
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java Mon May 6 11:10:40 2013
@@ -19,11 +19,11 @@ import org.apache.hadoop.hive.ql.parse.P
import org.apache.hadoop.hive.ql.parse.ParseUtils;
public class HQLParser {
-
+
public static interface ASTNodeVisitor {
public void visit(TreeNode node);
}
-
+
public static class TreeNode {
final TreeNode parent;
final ASTNode node;
@@ -31,18 +31,20 @@ public class HQLParser {
this.parent = parent;
this.node = node;
}
-
+
public TreeNode getParent() {
return parent;
}
-
+
public ASTNode getNode() {
return node;
}
}
-
+
public static final Set<Integer> BINARY_OPERATORS;
-
+ public static final Set<Integer> ARITHMETIC_OPERATORS;
+
+
static {
HashSet<Integer> ops = new HashSet<Integer>();
ops.add(DOT);
@@ -61,8 +63,21 @@ public class HQLParser {
ops.add(MOD);
ops.add(KW_LIKE);
BINARY_OPERATORS = Collections.unmodifiableSet(ops);
+
+
+ ARITHMETIC_OPERATORS = new HashSet<Integer>();
+ ARITHMETIC_OPERATORS.add(PLUS);
+ ARITHMETIC_OPERATORS.add(MINUS);
+ ARITHMETIC_OPERATORS.add(STAR);
+ ARITHMETIC_OPERATORS.add(DIVIDE);
+ ARITHMETIC_OPERATORS.add(MOD);
+
+ }
+
+ public static boolean isArithmeticOp(int tokenType) {
+ return ARITHMETIC_OPERATORS.contains(tokenType);
}
-
+
public static ASTNode parseHQL(String query) throws ParseException {
ParseDriver driver = new ParseDriver();
ASTNode tree = driver.parse(query);
@@ -70,7 +85,7 @@ public class HQLParser {
printAST(tree);
return tree;
}
-
+
public static void printAST(ASTNode node) {
try {
printAST(getHiveTokenMapping(), node, 0, 0);
@@ -78,7 +93,7 @@ public class HQLParser {
e.printStackTrace();
}
}
-
+
/**
* Debug function for printing query AST to stdout
* @param node
@@ -89,7 +104,7 @@ public class HQLParser {
if (node == null || node.isNil()) {
return;
}
-
+
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
@@ -97,10 +112,10 @@ public class HQLParser {
System.out.print(node.getText() + " [" + tokenMapping.get(
node.getToken().getType()) + "]");
System.out.print(" (l"+level + "c" + child + ")");
-
+
if (node.getChildCount() > 0) {
System.out.println(" {");
-
+
for (int i = 0; i < node.getChildCount(); i++) {
Tree tree = node.getChild(i);
if (tree instanceof ASTNode) {
@@ -110,21 +125,21 @@ public class HQLParser {
}
System.out.println();
}
-
+
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
-
+
System.out.print("}");
-
+
} else {
System.out.print('$');
}
}
-
+
public static Map<Integer, String> getHiveTokenMapping() throws Exception {
Map<Integer, String> mapping = new HashMap<Integer, String>();
-
+
for (Field f : HiveParser.class.getFields()) {
if (f.getType() == int.class) {
Integer tokenId = f.getInt(null);
@@ -132,15 +147,15 @@ public class HQLParser {
mapping.put(tokenId, token);
}
}
-
+
return mapping;
}
-
+
/**
* Find a node in the tree rooted at root, given the path of type of tokens
* from the root's children to the desired node
- *
+ *
* @param root
* @param path starts at the level of root's children
* @return
@@ -149,7 +164,7 @@ public class HQLParser {
for (int i = 0; i < path.length; i++) {
int type = path[i];
boolean hasChildWithType = false;
-
+
for (int j = 0; j < root.getChildCount(); j++) {
ASTNode node = (ASTNode) root.getChild(j);
if (node.getToken().getType() == type) {
@@ -167,16 +182,16 @@ public class HQLParser {
continue;
}
}
-
+
if (!hasChildWithType) {
// No path from this level
break;
}
}
-
+
return null;
}
-
+
/**
* Breadth first traversal of AST
* @param root
@@ -186,13 +201,13 @@ public class HQLParser {
if (root == null) {
throw new NullPointerException("Root cannot be null");
}
-
+
if (visitor == null) {
throw new NullPointerException("Visitor cannot be null");
}
Queue<TreeNode> queue = new LinkedList<TreeNode>();
queue.add(new TreeNode(null, root));
-
+
while (!queue.isEmpty()) {
TreeNode node = queue.poll();
visitor.visit(node);
@@ -202,7 +217,7 @@ public class HQLParser {
}
}
}
-
+
/**
* Recursively reconstruct query string given a query AST
* @param root
@@ -241,18 +256,18 @@ public class HQLParser {
if (i != root.getChildCount() -1) {
buf.append(", ");
}
- }
+ }
} else {
for (int i = 0; i < root.getChildCount(); i++) {
toInfixString((ASTNode) root.getChild(i), buf);
}
}
}
-
+
public static void main(String[] args) throws Exception {
- ASTNode ast = parseHQL("select * from default_table "
+ ASTNode ast = parseHQL("select * from default_table "
);
-
+
printAST(getHiveTokenMapping(), ast, 0, 0);
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java Mon May 6 11:10:40 2013
@@ -17,7 +17,7 @@ public class LeastDimensionResolver impl
@Override
public void rewriteContext(CubeQueryContext cubeql)
throws SemanticException {
- if (cubeql.getCube() != null) {
+ if (cubeql.getCube() != null && !cubeql.getCandidateFactTables().isEmpty()) {
Map<CubeFactTable, Integer> dimWeightMap =
new HashMap<CubeFactTable, Integer>();
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java Mon May 6 11:10:40 2013
@@ -19,7 +19,7 @@ public class LeastPartitionResolver impl
@Override
public void rewriteContext(CubeQueryContext cubeql)
throws SemanticException {
- if (cubeql.getCube() != null) {
+ if (cubeql.getCube() != null && !cubeql.getCandidateFactTables().isEmpty()) {
Map<CubeFactTable, Integer> numPartitionsMap =
new HashMap<CubeFactTable, Integer>();
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java Mon May 6 11:10:40 2013
@@ -32,9 +32,10 @@ public class PartitionResolver implement
CubeFactTable fact = i.next();
Map<UpdatePeriod, List<String>> partitionColMap =
new HashMap<UpdatePeriod, List<String>>();
- factPartitionMap.put(fact, partitionColMap);
if (!getPartitions(fact, fromDate, toDate, partitionColMap, cubeql)) {
i.remove();
+ } else {
+ factPartitionMap.put(fact, partitionColMap);
}
}
// set partition cols map in cubeql
@@ -51,7 +52,7 @@ public class PartitionResolver implement
UpdatePeriod interval = fact.maxIntervalInRange(fromDate, toDate);
if (interval == null) {
- System.err.println("Could not find a partition for given range:"
+ System.err.println("Could not find partition for given range:"
+ fromDate + "-" + toDate + " in fact:" + fact.getName());
return false;
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Mon May 6 11:10:40 2013
@@ -1,6 +1,7 @@
package org.apache.hadoop.hive.ql.cube.processors;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
@@ -274,4 +275,40 @@ public class TestCubeDriver {
System.out.println("cube hql:" + hqlQuery);
}
+ @Test
+ public void testAggregateResolver() throws Exception {
+ conf = new Configuration();
+ driver = new CubeDriver(new HiveConf(new Configuration(), HiveConf.class));
+
+ String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
+ System.out.println("#$AGGREGATE_RESOLVER_ TIME_RANGE:" + timeRange);
+ String q1 = "SELECT countryid, testCube.msr2 from testCube " + timeRange;
+ String q2 = "SELECT countryid, testCube.msr2 * testCube.msr2 from testCube " + timeRange;
+ String q3 = "SELECT countryid, sum(testCube.msr2) from testCube " + timeRange;
+ String q4 = "SELECT countryid, sum(testCube.msr2) from testCube " + timeRange
+ + " having testCube.msr2 > 100";
+ String q5 = "SELECT countryid, testCube.msr2 from testCube " + timeRange
+ + " having testCube.msr2 + testCube.msr2 > 100";
+ String q6 = "SELECT countryid, testCube.msr2 from testCube " + timeRange
+ + " having testCube.msr2 > 100 AND testCube.msr2 < 100";
+ String q7 = "SELECT countryid, sum(testCube.msr2) from testCube " + timeRange
+ + " having (testCube.msr2 > 100) OR (testcube.msr2 < 100 AND SUM(testcube.msr3) > 1000)";
+
+ String tests[] = {q1, q2, q3, q4, q5, q6, q7};
+
+ int exceptions[] = new int[tests.length];
+ for (int i = 0; i < tests.length; i++) {
+ String hql = null;
+ try {
+ hql = driver.compileCubeQuery(tests[i]);
+ } catch (SemanticException exc) {
+ exceptions[i] = i;
+ exc.printStackTrace();
+ }
+ System.out.println("##----AGGREGATE_RESOLVER_CUBEQL----#" + i + " [" + tests[i] + " ]");
+ System.out.println("##----AGGREGATE_RESOLVER_HQL-----#" + i + " [ " + hql + " ]");
+ }
+ System.out.println("##---AGGREGATE_RESOLVER_ exceptions=" + Arrays.toString(exceptions) );
+ }
+
}