You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/02/17 07:49:34 UTC
svn commit: r1660293 [15/48] - in /hive/branches/spark: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/
accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/
accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/ accumul...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Tue Feb 17 06:49:27 2015
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -68,6 +69,9 @@ import org.apache.hadoop.hive.ql.plan.Se
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef;
import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.ptf.Noop;
@@ -260,81 +264,119 @@ public final class ColumnPrunerProcFacto
PTFDesc conf = op.getConf();
//Since we cannot know what columns will be needed by a PTF chain,
//we do not prune columns on PTFOperator for PTF chains.
- if (!conf.forWindowing() && !Noop.class.isInstance(conf.getFuncDef().getTFunction())) {
+ PartitionedTableFunctionDef funcDef = conf.getFuncDef();
+ if (!conf.forWindowing() && !Noop.class.isInstance(funcDef.getTFunction())) {
return super.process(nd, stack, cppCtx, nodeOutputs);
}
-
- List<String> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0));
-
- WindowTableFunctionDef def = null;
- if (conf.forWindowing()) {
- def = (WindowTableFunctionDef) conf.getFuncDef();
- prunedCols = prunedColumnsList(prunedCols, def);
+
+ //we create a copy of prunedCols to create a list of pruned columns for PTFOperator
+ List<String> prunedCols =
+ new ArrayList<String>(cppCtx.getPrunedColList(op.getChildOperators().get(0)));
+ if (funcDef instanceof WindowTableFunctionDef) {
+ WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef;
+ prunedCols = Utilities.mergeUniqElems(getWindowFunctionColumns(def), prunedCols);
}
+
+ List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef);
+
+ op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS));
- RowSchema oldRS = op.getSchema();
- ArrayList<ColumnInfo> sig = buildPrunedRR(prunedCols, oldRS);
- op.getSchema().setSignature(sig);
-
- prunedCols = def == null ? prunedCols : prunedInputList(prunedCols, def);
- cppCtx.getPrunedColLists().put(op, prunedCols);
+ ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape();
+ cppCtx.getPrunedColLists().put(op, outputShape.getColumnNames());
return null;
}
- private static ArrayList<ColumnInfo> buildPrunedRR(List<String> prunedCols,
- RowSchema oldRS) throws SemanticException{
+ private List<ColumnInfo> buildPrunedRS(List<String> prunedCols, RowSchema oldRS)
+ throws SemanticException {
ArrayList<ColumnInfo> sig = new ArrayList<ColumnInfo>();
HashSet<String> prunedColsSet = new HashSet<String>(prunedCols);
- for(ColumnInfo cInfo : oldRS.getSignature()) {
- if ( prunedColsSet.contains(cInfo.getInternalName())) {
+ for (ColumnInfo cInfo : oldRS.getSignature()) {
+ if (prunedColsSet.contains(cInfo.getInternalName())) {
sig.add(cInfo);
}
}
return sig;
}
+ // always should be in this order (see PTFDeserializer#initializeWindowing)
+ private List<String> getWindowFunctionColumns(WindowTableFunctionDef tDef) {
+ List<String> columns = new ArrayList<String>();
+ if (tDef.getWindowFunctions() != null) {
+ for (WindowFunctionDef wDef : tDef.getWindowFunctions()) {
+ columns.add(wDef.getAlias());
+ }
+ }
+ return columns;
+ }
+
+ private RowResolver buildPrunedRR(List<String> prunedCols, RowSchema oldRS)
+ throws SemanticException {
+ RowResolver resolver = new RowResolver();
+ HashSet<String> prunedColsSet = new HashSet<String>(prunedCols);
+ for (ColumnInfo cInfo : oldRS.getSignature()) {
+ if (prunedColsSet.contains(cInfo.getInternalName())) {
+ resolver.put(cInfo.getTabAlias(), cInfo.getAlias(), cInfo);
+ }
+ }
+ return resolver;
+ }
+
/*
* add any input columns referenced in WindowFn args or expressions.
*/
- private ArrayList<String> prunedColumnsList(List<String> prunedCols,
- WindowTableFunctionDef tDef) {
- //we create a copy of prunedCols to create a list of pruned columns for PTFOperator
- ArrayList<String> mergedColList = new ArrayList<String>(prunedCols);
- if ( tDef.getWindowFunctions() != null ) {
- for(WindowFunctionDef wDef : tDef.getWindowFunctions() ) {
- if ( wDef.getArgs() == null) {
- continue;
+ private List<ColumnInfo> prunedColumnsList(List<String> prunedCols, RowSchema oldRS,
+ PartitionedTableFunctionDef pDef) throws SemanticException {
+ pDef.getOutputShape().setRr(null);
+ pDef.getOutputShape().setColumnNames(null);
+ if (pDef instanceof WindowTableFunctionDef) {
+ WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef;
+ if (tDef.getWindowFunctions() != null) {
+ for (WindowFunctionDef wDef : tDef.getWindowFunctions()) {
+ if (wDef.getArgs() == null) {
+ continue;
+ }
+ for (PTFExpressionDef arg : wDef.getArgs()) {
+ ExprNodeDesc exprNode = arg.getExprNode();
+ Utilities.mergeUniqElems(prunedCols, exprNode.getCols());
+ }
}
- for(PTFExpressionDef arg : wDef.getArgs()) {
- ExprNodeDesc exprNode = arg.getExprNode();
- Utilities.mergeUniqElems(mergedColList, exprNode.getCols());
+ }
+ if (tDef.getPartition() != null) {
+ for (PTFExpressionDef col : tDef.getPartition().getExpressions()) {
+ ExprNodeDesc exprNode = col.getExprNode();
+ Utilities.mergeUniqElems(prunedCols, exprNode.getCols());
}
}
+ if (tDef.getOrder() != null) {
+ for (PTFExpressionDef col : tDef.getOrder().getExpressions()) {
+ ExprNodeDesc exprNode = col.getExprNode();
+ Utilities.mergeUniqElems(prunedCols, exprNode.getCols());
+ }
+ }
+ } else {
+ pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS));
}
- if(tDef.getPartition() != null){
- for(PTFExpressionDef col : tDef.getPartition().getExpressions()){
- ExprNodeDesc exprNode = col.getExprNode();
- Utilities.mergeUniqElems(mergedColList, exprNode.getCols());
- }
- }
- if(tDef.getOrder() != null){
- for(PTFExpressionDef col : tDef.getOrder().getExpressions()){
- ExprNodeDesc exprNode = col.getExprNode();
- Utilities.mergeUniqElems(mergedColList, exprNode.getCols());
- }
- }
- return mergedColList;
+
+ PTFInputDef input = pDef.getInput();
+ if (input instanceof PartitionedTableFunctionDef) {
+ return prunedColumnsList(prunedCols, oldRS, (PartitionedTableFunctionDef)input);
+ }
+
+ ArrayList<String> inputColumns = prunedInputList(prunedCols, input);
+ input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRS));
+ input.getOutputShape().setColumnNames(inputColumns);
+
+ return buildPrunedRS(prunedCols, oldRS);
}
/*
* from the prunedCols list filter out columns that refer to WindowFns or WindowExprs
* the returned list is set as the prunedList needed by the PTFOp.
*/
- private ArrayList<String> prunedInputList(List<String> prunedCols,
- WindowTableFunctionDef tDef) {
+ private ArrayList<String> prunedInputList(List<String> prunedCols, PTFInputDef tDef) {
ArrayList<String> prunedInputCols = new ArrayList<String>();
- StructObjectInspector OI = tDef.getInput().getOutputShape().getOI();
+ StructObjectInspector OI = tDef.getOutputShape().getOI();
for(StructField f : OI.getAllStructFieldRefs()) {
String fName = f.getFieldName();
if ( prunedCols.contains(fName)) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Tue Feb 17 06:49:27 2015
@@ -90,16 +90,25 @@ public class GenMRTableScan1 implements
QBParseInfo parseInfo = parseCtx.getQB().getParseInfo();
if (parseInfo.isAnalyzeCommand()) {
- boolean partialScan = parseInfo.isPartialScanAnalyzeCommand();
- boolean noScan = parseInfo.isNoScanAnalyzeCommand();
- if (inputFormat.equals(OrcInputFormat.class) && (noScan || partialScan)) {
-
+ if (inputFormat.equals(OrcInputFormat.class)) {
+ // For ORC, all the following statements are the same
+ // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
+
// There will not be any MR or Tez job above this task
StatsNoJobWork snjWork = new StatsNoJobWork(parseCtx.getQB().getParseInfo().getTableSpec());
snjWork.setStatsReliable(parseCtx.getConf().getBoolVar(
HiveConf.ConfVars.HIVE_STATS_RELIABLE));
+ // If partition is specified, get pruned partition list
+ Set<Partition> confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+ if (confirmedParts.size() > 0) {
+ Table source = parseCtx.getQB().getMetaData().getTableForAlias(alias);
+ List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+ PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts,
+ partCols, false);
+ snjWork.setPrunedPartitionList(partList);
+ }
Task<StatsNoJobWork> snjTask = TaskFactory.get(snjWork, parseCtx.getConf());
ctx.setCurrTask(snjTask);
ctx.setCurrTopOp(null);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Tue Feb 17 06:49:27 2015
@@ -90,7 +90,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileMergeDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
@@ -586,7 +586,7 @@ public final class GenMapRedUtils {
// Later the properties have to come from the partition as opposed
// to from the table in order to support versioning.
Path[] paths = null;
- sampleDesc sampleDescr = parseCtx.getOpToSamplePruner().get(topOp);
+ SampleDesc sampleDescr = parseCtx.getOpToSamplePruner().get(topOp);
// Lookup list bucketing pruner
Map<String, ExprNodeDesc> partToPruner = parseCtx.getOpToPartToSkewedPruner().get(topOp);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java Tue Feb 17 06:49:27 2015
@@ -18,13 +18,19 @@
package org.apache.hadoop.hive.ql.optimizer;
+import java.util.Collection;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.LimitOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -32,12 +38,16 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.parse.GlobalLimitCtx;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.QB;
-import org.apache.hadoop.hive.ql.parse.QBParseInfo;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.SplitSample;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
/**
* This optimizer is used to reduce the input size for the query for queries which are
@@ -63,9 +73,6 @@ public class GlobalLimitOptimizer implem
Map<TableScanOperator, ExprNodeDesc> opToPartPruner = pctx.getOpToPartPruner();
Map<String, SplitSample> nameToSplitSample = pctx.getNameToSplitSample();
- QB qb = pctx.getQB();
- QBParseInfo qbParseInfo = qb.getParseInfo();
-
// determine the query qualifies reduce input size for LIMIT
// The query only qualifies when there are only one top operator
// and there is no transformer or UDTF and no block sampling
@@ -86,15 +93,17 @@ public class GlobalLimitOptimizer implem
// FROM ... LIMIT...
// SELECT * FROM (SELECT col1 as col2 (SELECT * FROM ...) t1 LIMIT ...) t2);
//
- Integer tempGlobalLimit = checkQbpForGlobalLimit(qb);
+ TableScanOperator ts = (TableScanOperator) topOps.values().toArray()[0];
+ Integer tempGlobalLimit = checkQbpForGlobalLimit(ts);
// query qualify for the optimization
if (tempGlobalLimit != null && tempGlobalLimit != 0) {
- TableScanOperator ts = (TableScanOperator) topOps.values().toArray()[0];
Table tab = ts.getConf().getTableMetadata();
if (!tab.isPartitioned()) {
- if (qbParseInfo.getDestToWhereExpr().isEmpty()) {
+ Set<FilterOperator> filterOps =
+ OperatorUtils.findOperators(ts, FilterOperator.class);
+ if (filterOps.size() == 0) {
globalLimitCtx.enableOpt(tempGlobalLimit);
}
} else {
@@ -130,49 +139,59 @@ public class GlobalLimitOptimizer implem
}
/**
- * Recursively check the limit number in all sub queries
+ * Check the limit number in all sub queries
*
- * @param qbParseInfo
* @return if there is one and only one limit for all subqueries, return the limit
* if there is no limit, return 0
* otherwise, return null
*/
- private Integer checkQbpForGlobalLimit(QB localQb) {
- QBParseInfo qbParseInfo = localQb.getParseInfo();
- if (localQb.getNumSelDi() == 0 && qbParseInfo.getDestToClusterBy().isEmpty()
- && qbParseInfo.getDestToDistributeBy().isEmpty()
- && qbParseInfo.getDestToOrderBy().isEmpty()
- && qbParseInfo.getDestToSortBy().isEmpty()
- && qbParseInfo.getDestToAggregationExprs().size() <= 1
- && qbParseInfo.getDestToDistinctFuncExprs().size() <= 1
- && qbParseInfo.getNameToSample().isEmpty()) {
- if ((qbParseInfo.getDestToAggregationExprs().size() < 1 ||
- qbParseInfo.getDestToAggregationExprs().values().iterator().next().isEmpty()) &&
- (qbParseInfo.getDestToDistinctFuncExprs().size() < 1 ||
- qbParseInfo.getDestToDistinctFuncExprs().values().iterator().next().isEmpty())
- && qbParseInfo.getDestToLimit().size() <= 1) {
- Integer retValue;
- if (qbParseInfo.getDestToLimit().size() == 0) {
- retValue = 0;
- } else {
- retValue = qbParseInfo.getDestToLimit().values().iterator().next();
- }
-
- for (String alias : localQb.getSubqAliases()) {
- Integer limit = checkQbpForGlobalLimit(localQb.getSubqForAlias(alias).getQB());
- if (limit == null) {
- return null;
- } else if (retValue > 0 && limit > 0) {
- // Any query has more than one LIMITs shown in the query is not
- // qualified to this optimization
- return null;
- } else if (limit > 0) {
- retValue = limit;
- }
- }
- return retValue;
+ private static Integer checkQbpForGlobalLimit(TableScanOperator ts) {
+ Set<Class<? extends Operator<?>>> searchedClasses =
+ new ImmutableSet.Builder<Class<? extends Operator<?>>>()
+ .add(ReduceSinkOperator.class)
+ .add(GroupByOperator.class)
+ .add(FilterOperator.class)
+ .add(LimitOperator.class)
+ .build();
+ Multimap<Class<? extends Operator<?>>, Operator<?>> ops =
+ OperatorUtils.classifyOperators(ts, searchedClasses);
+ // To apply this optimization, in the input query:
+ // - There cannot exist any order by/sort by clause,
+ // thus existsOrdering should be false.
+ // - There cannot exist any distribute by clause, thus
+ // existsPartitioning should be false.
+ // - There cannot exist any cluster by clause, thus
+ // existsOrdering AND existsPartitioning should be false.
+ for (Operator<?> op : ops.get(ReduceSinkOperator.class)) {
+ ReduceSinkDesc reduceSinkConf = ((ReduceSinkOperator) op).getConf();
+ if (reduceSinkConf.isOrdering() || reduceSinkConf.isPartitioning()) {
+ return null;
}
}
+ // - There cannot exist any (distinct) aggregate.
+ for (Operator<?> op : ops.get(GroupByOperator.class)) {
+ GroupByDesc groupByConf = ((GroupByOperator) op).getConf();
+ if (groupByConf.isAggregate() || groupByConf.isDistinct()) {
+ return null;
+ }
+ }
+ // - There cannot exist any sampling predicate.
+ for (Operator<?> op : ops.get(FilterOperator.class)) {
+ FilterDesc filterConf = ((FilterOperator) op).getConf();
+ if (filterConf.getIsSamplingPred()) {
+ return null;
+ }
+ }
+ // If there is one and only one limit starting at op, return the limit
+ // If there is no limit, return 0
+ // Otherwise, return null
+ Collection<Operator<?>> limitOps = ops.get(LimitOperator.class);
+ if (limitOps.size() == 1) {
+ return ((LimitOperator) limitOps.iterator().next()).getConf().getLimit();
+ }
+ else if (limitOps.size() == 0) {
+ return 0;
+ }
return null;
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java Tue Feb 17 06:49:27 2015
@@ -190,7 +190,7 @@ public final class IndexUtils {
List<Index> indexesOnTable;
try {
- indexesOnTable = baseTableMetaData.getAllIndexes((short) -1); // get all indexes
+ indexesOnTable = getAllIndexes(baseTableMetaData, (short) -1); // get all indexes
} catch (HiveException e) {
throw new SemanticException("Error accessing metastore", e);
}
@@ -204,6 +204,14 @@ public final class IndexUtils {
return matchingIndexes;
}
+ /**
+ * @return List containing Indexes names if there are indexes on this table
+ * @throws HiveException
+ **/
+ public static List<Index> getAllIndexes(Table table, short max) throws HiveException {
+ Hive hive = Hive.get();
+ return hive.getIndexes(table.getTTable().getDbName(), table.getTTable().getTableName(), max);
+ }
public static Task<?> createRootTask(HiveConf builderConf, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, StringBuilder command,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java Tue Feb 17 06:49:27 2015
@@ -48,7 +48,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
/**
* The transformation step that does sample pruning.
@@ -61,17 +61,17 @@ public class SamplePruner implements Tra
*
*/
public static class SamplePrunerCtx implements NodeProcessorCtx {
- HashMap<TableScanOperator, sampleDesc> opToSamplePruner;
+ HashMap<TableScanOperator, SampleDesc> opToSamplePruner;
public SamplePrunerCtx(
- HashMap<TableScanOperator, sampleDesc> opToSamplePruner) {
+ HashMap<TableScanOperator, SampleDesc> opToSamplePruner) {
this.opToSamplePruner = opToSamplePruner;
}
/**
* @return the opToSamplePruner
*/
- public HashMap<TableScanOperator, sampleDesc> getOpToSamplePruner() {
+ public HashMap<TableScanOperator, SampleDesc> getOpToSamplePruner() {
return opToSamplePruner;
}
@@ -80,7 +80,7 @@ public class SamplePruner implements Tra
* the opToSamplePruner to set
*/
public void setOpToSamplePruner(
- HashMap<TableScanOperator, sampleDesc> opToSamplePruner) {
+ HashMap<TableScanOperator, SampleDesc> opToSamplePruner) {
this.opToSamplePruner = opToSamplePruner;
}
}
@@ -135,7 +135,7 @@ public class SamplePruner implements Tra
Object... nodeOutputs) throws SemanticException {
FilterOperator filOp = (FilterOperator) nd;
FilterDesc filOpDesc = filOp.getConf();
- sampleDesc sampleDescr = filOpDesc.getSampleDescr();
+ SampleDesc sampleDescr = filOpDesc.getSampleDescr();
if ((sampleDescr == null) || !sampleDescr.getInputPruning()) {
return null;
@@ -182,7 +182,7 @@ public class SamplePruner implements Tra
* @throws SemanticException
*/
@SuppressWarnings("nls")
- public static Path[] prune(Partition part, sampleDesc sampleDescr)
+ public static Path[] prune(Partition part, SampleDesc sampleDescr)
throws SemanticException {
int num = sampleDescr.getNumerator();
int den = sampleDescr.getDenominator();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Tue Feb 17 06:49:27 2015
@@ -34,7 +34,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -74,8 +73,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hive.common.util.AnnotationUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.thrift.TException;
import com.google.common.collect.Lists;
@@ -143,6 +141,22 @@ public class StatsOptimizer implements T
Unsupported
}
+ enum LongSubType {
+ BIGINT { Object cast(long longValue) { return longValue; } },
+ INT { Object cast(long longValue) { return (int)longValue; } },
+ SMALLINT { Object cast(long longValue) { return (short)longValue; } },
+ TINYINT { Object cast(long longValue) { return (byte)longValue; } };
+
+ abstract Object cast(long longValue);
+ }
+
+ enum DoubleSubType {
+ DOUBLE { Object cast(double doubleValue) { return doubleValue; } },
+ FLOAT { Object cast(double doubleValue) { return (float) doubleValue; } };
+
+ abstract Object cast(double doubleValue);
+ }
+
private StatType getType(String origType) {
if (serdeConstants.IntegralTypes.contains(origType)) {
return StatType.Integeral;
@@ -190,57 +204,56 @@ public class StatsOptimizer implements T
try {
TableScanOperator tsOp = (TableScanOperator) stack.get(0);
- if(tsOp.getParentOperators() != null && tsOp.getParentOperators().size() > 0) {
+ if (tsOp.getNumParent() > 0) {
// looks like a subq plan.
return null;
}
- SelectOperator selOp = (SelectOperator)tsOp.getChildren().get(0);
- for(ExprNodeDesc desc : selOp.getConf().getColList()) {
+ SelectOperator pselOp = (SelectOperator)stack.get(1);
+ for(ExprNodeDesc desc : pselOp.getConf().getColList()) {
if (!((desc instanceof ExprNodeColumnDesc) || (desc instanceof ExprNodeConstantDesc))) {
// Probably an expression, cant handle that
return null;
}
}
- Map<String, ExprNodeDesc> exprMap = selOp.getColumnExprMap();
- // Since we have done an exact match on TS-SEL-GBY-RS-GBY-SEL-FS
+ Map<String, ExprNodeDesc> exprMap = pselOp.getColumnExprMap();
+ // Since we have done an exact match on TS-SEL-GBY-RS-GBY-(SEL)-FS
// we need not to do any instanceof checks for following.
- GroupByOperator gbyOp = (GroupByOperator)selOp.getChildren().get(0);
- ReduceSinkOperator rsOp = (ReduceSinkOperator)gbyOp.getChildren().get(0);
+ GroupByOperator pgbyOp = (GroupByOperator)stack.get(2);
+ if (pgbyOp.getConf().getOutputColumnNames().size() !=
+ pgbyOp.getConf().getAggregators().size()) {
+ return null;
+ }
+ ReduceSinkOperator rsOp = (ReduceSinkOperator)stack.get(3);
if (rsOp.getConf().getDistinctColumnIndices().size() > 0) {
// we can't handle distinct
return null;
}
- Operator<?> last = rsOp.getChildOperators().get(0);
- if (last.getChildOperators().get(0) instanceof SelectOperator) {
- selOp = (SelectOperator)rsOp.getChildOperators().get(0).getChildOperators().get(0);
- last = selOp;
- if (!(selOp.getConf().getColList().size() ==
- gbyOp.getConf().getAggregators().size())) {
- // all select columns must be aggregations
- return null;
-
- }
- for(ExprNodeDesc desc : selOp.getConf().getColList()) {
- if (!(desc instanceof ExprNodeColumnDesc)) {
- // Probably an expression, cant handle that
- return null;
- }
+ GroupByOperator cgbyOp = (GroupByOperator)stack.get(4);
+ if (cgbyOp.getConf().getOutputColumnNames().size() !=
+ cgbyOp.getConf().getAggregators().size()) {
+ return null;
+ }
+ Operator<?> last = (Operator<?>) stack.get(5);
+ if (last instanceof SelectOperator) {
+ SelectOperator cselOp = (SelectOperator) last;
+ if (!cselOp.isIdentitySelect()) {
+ return null; // todo we can do further by providing operator to fetch task
}
+ last = (Operator<?>) stack.get(6);
}
- FileSinkOperator fsOp = (FileSinkOperator)(last.getChildren().get(0));
- if (fsOp.getChildOperators() != null && fsOp.getChildOperators().size() > 0) {
+ FileSinkOperator fsOp = (FileSinkOperator)last;
+ if (fsOp.getNumChild() > 0) {
// looks like a subq plan.
- return null;
+ return null; // todo we can collapse this part of tree into single TS
}
Table tbl = tsOp.getConf().getTableMetadata();
List<Object> oneRow = new ArrayList<Object>();
- List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
Hive hive = Hive.get(pctx.getConf());
- for (AggregationDesc aggr : gbyOp.getConf().getAggregators()) {
+ for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) {
if (aggr.getDistinct()) {
// our stats for NDV is approx, not accurate.
return null;
@@ -249,7 +262,12 @@ public class StatsOptimizer implements T
GenericUDAFResolver udaf =
FunctionRegistry.getGenericUDAFResolver(aggr.getGenericUDAFName());
if (udaf instanceof GenericUDAFSum) {
+ // long/double/decimal
ExprNodeDesc desc = aggr.getParameters().get(0);
+ PrimitiveCategory category = GenericUDAFSum.getReturnType(desc.getTypeInfo());
+ if (category == null) {
+ return null;
+ }
String constant;
if (desc instanceof ExprNodeConstantDesc) {
constant = ((ExprNodeConstantDesc) desc).getValue().toString();
@@ -262,11 +280,22 @@ public class StatsOptimizer implements T
if(rowCnt == null) {
return null;
}
- oneRow.add(HiveDecimal.create(constant).multiply(HiveDecimal.create(rowCnt)));
- ois.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
- PrimitiveCategory.DECIMAL));
+ switch (category) {
+ case LONG:
+ oneRow.add(Long.valueOf(constant) * rowCnt);
+ break;
+ case DOUBLE:
+ oneRow.add(Double.valueOf(constant) * rowCnt);
+ break;
+ case DECIMAL:
+ oneRow.add(HiveDecimal.create(constant).multiply(HiveDecimal.create(rowCnt)));
+ break;
+ default:
+ throw new IllegalStateException("never");
+ }
}
else if (udaf instanceof GenericUDAFCount) {
+ // always long
Long rowCnt = 0L;
if (aggr.getParameters().isEmpty() || aggr.getParameters().get(0) instanceof
ExprNodeConstantDesc || ((aggr.getParameters().get(0) instanceof ExprNodeColumnDesc) &&
@@ -341,8 +370,6 @@ public class StatsOptimizer implements T
}
}
oneRow.add(rowCnt);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
} else if (udaf instanceof GenericUDAFMax) {
ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
String colName = colDesc.getColumn();
@@ -359,19 +386,28 @@ public class StatsOptimizer implements T
return null;
}
ColumnStatisticsData statData = stats.get(0).getStatsData();
+ String name = colDesc.getTypeString().toUpperCase();
switch (type) {
- case Integeral:
+ case Integeral: {
+ LongSubType subType = LongSubType.valueOf(name);
LongColumnStatsData lstats = statData.getLongStats();
- oneRow.add(lstats.isSetHighValue() ? lstats.getHighValue() : null);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
+ if (lstats.isSetHighValue()) {
+ oneRow.add(subType.cast(lstats.getHighValue()));
+ } else {
+ oneRow.add(null);
+ }
break;
- case Double:
+ }
+ case Double: {
+ DoubleSubType subType = DoubleSubType.valueOf(name);
DoubleColumnStatsData dstats = statData.getDoubleStats();
- oneRow.add(dstats.isSetHighValue() ? dstats.getHighValue() : null);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE));
+ if (dstats.isSetHighValue()) {
+ oneRow.add(subType.cast(dstats.getHighValue()));
+ } else {
+ oneRow.add(null);
+ }
break;
+ }
default:
// unsupported type
Log.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " +
@@ -381,8 +417,11 @@ public class StatsOptimizer implements T
} else {
Set<Partition> parts = pctx.getPrunedPartitions(
tsOp.getConf().getAlias(), tsOp).getPartitions();
+ String name = colDesc.getTypeString().toUpperCase();
switch (type) {
case Integeral: {
+ LongSubType subType = LongSubType.valueOf(name);
+
Long maxVal = null;
Collection<List<ColumnStatisticsObj>> result =
verifyAndGetPartStats(hive, tbl, colName, parts);
@@ -399,12 +438,16 @@ public class StatsOptimizer implements T
long curVal = lstats.getHighValue();
maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal);
}
- oneRow.add(maxVal);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
+ if (maxVal != null) {
+ oneRow.add(subType.cast(maxVal));
+ } else {
+ oneRow.add(maxVal);
+ }
break;
}
case Double: {
+ DoubleSubType subType = DoubleSubType.valueOf(name);
+
Double maxVal = null;
Collection<List<ColumnStatisticsObj>> result =
verifyAndGetPartStats(hive, tbl, colName, parts);
@@ -421,9 +464,11 @@ public class StatsOptimizer implements T
double curVal = statData.getDoubleStats().getHighValue();
maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal);
}
- oneRow.add(maxVal);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE));
+ if (maxVal != null) {
+ oneRow.add(subType.cast(maxVal));
+ } else {
+ oneRow.add(null);
+ }
break;
}
default:
@@ -444,19 +489,28 @@ public class StatsOptimizer implements T
ColumnStatisticsData statData = hive.getMSC().getTableColumnStatistics(
tbl.getDbName(), tbl.getTableName(), Lists.newArrayList(colName))
.get(0).getStatsData();
+ String name = colDesc.getTypeString().toUpperCase();
switch (type) {
- case Integeral:
+ case Integeral: {
+ LongSubType subType = LongSubType.valueOf(name);
LongColumnStatsData lstats = statData.getLongStats();
- oneRow.add(lstats.isSetLowValue() ? lstats.getLowValue() : null);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
+ if (lstats.isSetLowValue()) {
+ oneRow.add(subType.cast(lstats.getLowValue()));
+ } else {
+ oneRow.add(null);
+ }
break;
- case Double:
+ }
+ case Double: {
+ DoubleSubType subType = DoubleSubType.valueOf(name);
DoubleColumnStatsData dstats = statData.getDoubleStats();
- oneRow.add(dstats.isSetLowValue() ? dstats.getLowValue() : null);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE));
+ if (dstats.isSetLowValue()) {
+ oneRow.add(subType.cast(dstats.getLowValue()));
+ } else {
+ oneRow.add(null);
+ }
break;
+ }
default: // unsupported type
Log.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " +
"metadata optimizer for column : " + colName);
@@ -464,8 +518,11 @@ public class StatsOptimizer implements T
}
} else {
Set<Partition> parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions();
+ String name = colDesc.getTypeString().toUpperCase();
switch(type) {
case Integeral: {
+ LongSubType subType = LongSubType.valueOf(name);
+
Long minVal = null;
Collection<List<ColumnStatisticsObj>> result =
verifyAndGetPartStats(hive, tbl, colName, parts);
@@ -482,12 +539,16 @@ public class StatsOptimizer implements T
long curVal = lstats.getLowValue();
minVal = minVal == null ? curVal : Math.min(minVal, curVal);
}
- oneRow.add(minVal);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
+ if (minVal != null) {
+ oneRow.add(subType.cast(minVal));
+ } else {
+ oneRow.add(minVal);
+ }
break;
}
case Double: {
+ DoubleSubType subType = DoubleSubType.valueOf(name);
+
Double minVal = null;
Collection<List<ColumnStatisticsObj>> result =
verifyAndGetPartStats(hive, tbl, colName, parts);
@@ -504,9 +565,11 @@ public class StatsOptimizer implements T
double curVal = statData.getDoubleStats().getLowValue();
minVal = minVal == null ? curVal : Math.min(minVal, curVal);
}
- oneRow.add(minVal);
- ois.add(PrimitiveObjectInspectorFactory.
- getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE));
+ if (minVal != null) {
+ oneRow.add(subType.cast(minVal));
+ } else {
+ oneRow.add(minVal);
+ }
break;
}
default: // unsupported type
@@ -528,8 +591,10 @@ public class StatsOptimizer implements T
allRows.add(oneRow);
List<String> colNames = new ArrayList<String>();
- for (ColumnInfo colInfo: gbyOp.getSchema().getSignature()) {
+ List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+ for (ColumnInfo colInfo: cgbyOp.getSchema().getSignature()) {
colNames.add(colInfo.getInternalName());
+ ois.add(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(colInfo.getType()));
}
StandardStructObjectInspector sOI = ObjectInspectorFactory.
getStandardStructObjectInspector(colNames, ois);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java Tue Feb 17 06:49:27 2015
@@ -180,7 +180,7 @@ public class RelOptHiveTable extends Rel
}
// We have valid pruning expressions, only retrieve qualifying partitions
- ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true));
+ ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true, getRelOptSchema().getTypeFactory()));
partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache);
} catch (HiveException he) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java Tue Feb 17 06:49:27 2015
@@ -722,7 +722,7 @@ public class ASTConverter {
public ASTNode getUnionAllAST(ASTNode leftAST, ASTNode rightAST) {
- ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST)
+ ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNIONALL, "TOK_UNIONALL").add(leftAST)
.add(rightAST).node();
return unionTokAST;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java Tue Feb 17 06:49:27 2015
@@ -33,12 +33,15 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeUtil;
/*
* convert a RexNode to an ExprNodeDesc
@@ -48,8 +51,9 @@ public class ExprNodeConverter extends R
RelDataType rType;
String tabAlias;
boolean partitioningExpr;
+ private final RelDataTypeFactory dTFactory;
- public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitioningExpr) {
+ public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitioningExpr, RelDataTypeFactory dTFactory) {
super(true);
/*
* hb: 6/25/14 for now we only support expressions that only contain
@@ -62,6 +66,7 @@ public class ExprNodeConverter extends R
this.tabAlias = tabAlias;
this.rType = rType;
this.partitioningExpr = partitioningExpr;
+ this.dTFactory = dTFactory;
}
@Override
@@ -85,9 +90,15 @@ public class ExprNodeConverter extends R
args.add(operand.accept(this));
}
- // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the
- // exprnode
- if (ASTConverter.isFlat(call)) {
+ // If Call is a redundant cast then bail out. Ex: cast(true)BOOLEAN
+ if (call.isA(SqlKind.CAST)
+ && (call.operands.size() == 1)
+ && SqlTypeUtil.equalSansNullability(dTFactory, call.getType(),
+ call.operands.get(0).getType())) {
+ return args.get(0);
+ } else if (ASTConverter.isFlat(call)) {
+ // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the
+ // exprnode
ArrayList<ExprNodeDesc> tmpExprArgs = new ArrayList<ExprNodeDesc>();
tmpExprArgs.addAll(args.subList(0, 2));
gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()),
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java Tue Feb 17 06:49:27 2015
@@ -64,7 +64,7 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Maps;
public class SqlFunctionConverter {
- private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
+ private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
static final Map<String, SqlOperator> hiveToCalcite;
static final Map<SqlOperator, HiveToken> calciteToHiveToken;
@@ -79,7 +79,7 @@ public class SqlFunctionConverter {
public static SqlOperator getCalciteOperator(String funcTextName, GenericUDF hiveUDF,
ImmutableList<RelDataType> calciteArgTypes, RelDataType retType)
- throws CalciteSemanticException {
+ throws SemanticException {
// handle overloaded methods first
if (hiveUDF instanceof GenericUDFOPNegative) {
return SqlStdOperatorTable.UNARY_MINUS;
@@ -182,7 +182,8 @@ public class SqlFunctionConverter {
} catch (UDFArgumentException e) {
throw new RuntimeException(e);
}
- return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf);
+ return new FunctionInfo(
+ fi.isNative(), fi.getDisplayName(), (GenericUDF) udf, fi.getResources());
}
// TODO: 1) handle Agg Func Name translation 2) is it correct to add func
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java Tue Feb 17 06:49:27 2015
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hive.ql.parse.Q
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
@@ -45,33 +47,27 @@ import org.apache.hadoop.hive.ql.parse.S
*
*/
public final class RewriteParseContextGenerator {
- private static final Log LOG = LogFactory.getLog(RewriteParseContextGenerator.class.getName());
- private RewriteParseContextGenerator(){
- }
+ private static final Log LOG = LogFactory.getLog(RewriteParseContextGenerator.class.getName());
/**
- * Parse the input {@link String} command and generate a ASTNode tree.
+ * Parse the input {@link String} command and generate an operator tree.
* @param conf
* @param command
- * @return the parse context
* @throws SemanticException
*/
- public static ParseContext generateOperatorTree(HiveConf conf,
- String command) throws SemanticException{
- Context ctx;
- ParseContext subPCtx = null;
+ public static Operator<? extends OperatorDesc> generateOperatorTree(HiveConf conf,
+ String command) throws SemanticException {
+ Operator<? extends OperatorDesc> operatorTree;
try {
- ctx = new Context(conf);
+ Context ctx = new Context(conf);
ParseDriver pd = new ParseDriver();
ASTNode tree = pd.parse(command, ctx);
tree = ParseUtils.findRootNonNullToken(tree);
BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree);
assert(sem instanceof SemanticAnalyzer);
- doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx);
-
- subPCtx = ((SemanticAnalyzer) sem).getParseContext();
+ operatorTree = doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx);
LOG.info("Sub-query Semantic Analysis Completed");
} catch (IOException e) {
LOG.error("IOException in generating the operator " +
@@ -89,13 +85,12 @@ public final class RewriteParseContextGe
LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
throw new SemanticException(e.getMessage(), e);
}
- return subPCtx;
-
+ return operatorTree;
}
/**
* For the input ASTNode tree, perform a semantic analysis and check metadata
- * Generate a operator tree and return the {@link ParseContext} instance for the operator tree.
+ * Generate a operator tree and return it.
*
* @param ctx
* @param sem
@@ -103,7 +98,7 @@ public final class RewriteParseContextGe
* @return
* @throws SemanticException
*/
- private static void doSemanticAnalysis(SemanticAnalyzer sem,
+ private static Operator<?> doSemanticAnalysis(SemanticAnalyzer sem,
ASTNode ast, Context ctx) throws SemanticException {
QB qb = new QB(null, null, false);
ASTNode child = ast;
@@ -119,9 +114,10 @@ public final class RewriteParseContextGe
LOG.info("Completed getting MetaData in Sub-query Semantic Analysis");
LOG.info("Sub-query Abstract syntax tree: " + ast.toStringTree());
- sem.genPlan(qb);
+ Operator<?> operator = sem.genPlan(qb);
LOG.info("Sub-query Completed plan generation");
+ return operator;
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java Tue Feb 17 06:49:27 2015
@@ -23,7 +23,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -32,6 +31,7 @@ import org.apache.hadoop.hive.ql.exec.Co
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
@@ -267,15 +267,17 @@ public final class RewriteQueryUsingAggr
String selReplacementCommand = "select sum(`" + rewriteQueryCtx.getAggregateFunction() + "`)"
+ " from " + rewriteQueryCtx.getIndexName() + " group by "
+ rewriteQueryCtx.getIndexKey() + " ";
- // create a new ParseContext for the query to retrieve its operator tree,
- // and the required GroupByOperator from it
- ParseContext newDAGContext = RewriteParseContextGenerator.generateOperatorTree(
- rewriteQueryCtx.getParseContext().getConf(), selReplacementCommand);
+ // retrieve the operator tree for the query, and the required GroupByOperator from it
+ Operator<?> newOperatorTree = RewriteParseContextGenerator.generateOperatorTree(
+ rewriteQueryCtx.getParseContext().getConf(),
+ selReplacementCommand);
// we get our new GroupByOperator here
- Map<GroupByOperator, Set<String>> newGbyOpMap = newDAGContext.getGroupOpToInputTables();
- GroupByOperator newGbyOperator = newGbyOpMap.keySet().iterator().next();
- GroupByDesc oldConf = operator.getConf();
+ GroupByOperator newGbyOperator = OperatorUtils.findLastOperatorUpstream(
+ newOperatorTree, GroupByOperator.class);
+ if (newGbyOperator == null) {
+ throw new SemanticException("Error replacing GroupBy operator.");
+ }
// we need this information to set the correct colList, outputColumnNames
// in SelectOperator
@@ -297,6 +299,7 @@ public final class RewriteQueryUsingAggr
// Now the GroupByOperator has the new AggregationList;
// sum(`_count_of_indexed_key`)
// instead of count(indexed_key)
+ GroupByDesc oldConf = operator.getConf();
oldConf.setAggregators((ArrayList<AggregationDesc>) newAggrList);
operator.setConf(oldConf);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Feb 17 06:49:27 2015
@@ -1086,10 +1086,6 @@ public class Vectorizer implements Physi
return false;
}
LOG.info("Reduce GROUP BY mode is " + desc.getMode().name());
- if (desc.getGroupKeyNotReductionKey()) {
- LOG.info("Reduce vector mode not supported when group key is not reduction key");
- return false;
- }
if (!aggregatorsOutputIsPrimitive(desc.getAggregators(), isReduce)) {
LOG.info("Reduce vector mode only supported when aggregate outputs are primitive types");
return false;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Feb 17 06:49:27 2015
@@ -119,7 +119,7 @@ public class PartitionPruner implements
return false;
}
- // All columns of the expression must be parttioned columns
+ // All columns of the expression must be partitioned columns
List<ExprNodeDesc> children = expr.getChildren();
if (children != null) {
for (int i = 0; i < children.size(); i++) {
@@ -158,16 +158,18 @@ public class PartitionPruner implements
* cached result for the table
* @return the partition list for the table that satisfies the partition
* pruner condition.
- * @throws HiveException
+ * @throws SemanticException
*/
public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
HiveConf conf, String alias, Map<String, PrunedPartitionList> prunedPartitionsMap)
throws SemanticException {
- LOG.trace("Started pruning partiton");
- LOG.trace("dbname = " + tab.getDbName());
- LOG.trace("tabname = " + tab.getTableName());
- LOG.trace("prune Expression = " + prunerExpr == null ? "" : prunerExpr);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Started pruning partiton");
+ LOG.trace("dbname = " + tab.getDbName());
+ LOG.trace("tabname = " + tab.getTableName());
+ LOG.trace("prune Expression = " + (prunerExpr == null ? "" : prunerExpr));
+ }
String key = tab.getDbName() + "." + tab.getTableName() + ";";
@@ -198,15 +200,19 @@ public class PartitionPruner implements
// For null and true values, return every partition
if (!isFalseExpr(compactExpr)) {
// Non-strict mode, and all the predicates are on non-partition columns - get everything.
- LOG.debug("Filter " + oldFilter + " was null after compacting");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Filter " + oldFilter + " was null after compacting");
+ }
return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
} else {
return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(new ArrayList<Partition>()),
new ArrayList<String>(), false);
}
}
- LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
- + "; filter w/o compacting: " + oldFilter);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
+ + "; filter w/o compacting: " + oldFilter);
+ }
key = key + compactExpr.getExprString();
PrunedPartitionList ppList = prunedPartitionsMap.get(key);
@@ -342,7 +348,7 @@ public class PartitionPruner implements
if (!(expr instanceof ExprNodeGenericFuncDesc)) {
return false;
}
- if (!FunctionRegistry.isNativeFuncExpr((ExprNodeGenericFuncDesc)expr)) {
+ if (!FunctionRegistry.isBuiltInFuncExpr((ExprNodeGenericFuncDesc) expr)) {
return true;
}
for (ExprNodeDesc child : expr.getChildren()) {
@@ -506,12 +512,16 @@ public class PartitionPruner implements
if (isUnknown && values.contains(defaultPartitionName)) {
// Reject default partitions if we couldn't determine whether we should include it or not.
// Note that predicate would only contains partition column parts of original predicate.
- LOG.debug("skipping default/bad partition: " + partName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("skipping default/bad partition: " + partName);
+ }
partIter.remove();
continue;
}
hasUnknownPartitions |= isUnknown;
- LOG.debug("retained " + (isUnknown ? "unknown " : "") + "partition: " + partName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("retained " + (isUnknown ? "unknown " : "") + "partition: " + partName);
+ }
}
if (!inPlace) {
partNames.clear();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Feb 17 06:49:27 2015
@@ -78,12 +78,14 @@ import org.apache.calcite.rel.type.RelDa
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.rex.RexFieldCollation;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexWindowBound;
import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Schemas;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlExplainLevel;
@@ -716,7 +718,7 @@ public class CalcitePlanner extends Sema
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
- hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
+ hepPgmBldr.addRuleInstance(ProjectRemoveRule.NAME_CALC_INSTANCE);
hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
hepPgm = hepPgmBldr.build();
@@ -790,7 +792,7 @@ public class CalcitePlanner extends Sema
RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
- HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
+ HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY, true);
basePlan = fieldTrimmer.trim(basePlan);
// 5. Rerun PPD through Project as column pruning would have introduced DT
@@ -833,6 +835,11 @@ public class CalcitePlanner extends Sema
basePlan.getCluster().setMetadataProvider(
new CachingRelMetadataProvider(chainedProvider, planner));
+ // Executor is required for constant-reduction rules; see [CALCITE-566]
+ final RexExecutorImpl executor =
+ new RexExecutorImpl(Schemas.createDataContext(null));
+ basePlan.getCluster().getPlanner().setExecutor(executor);
+
planner.setRoot(basePlan);
optimizedRelNode = planner.findBestExp();
@@ -859,49 +866,30 @@ public class CalcitePlanner extends Sema
ASTNode tabref = getQB().getAliases().isEmpty() ? null : getQB().getParseInfo()
.getSrcForAlias(getQB().getAliases().get(0));
- for (Map.Entry<String, ColumnInfo> lEntry : leftmap.entrySet()) {
- String field = lEntry.getKey();
+
+ // 3. construct Union Output RR using original left & right Input
+ RowResolver unionoutRR = new RowResolver();
+
+ Iterator<Map.Entry<String, ColumnInfo>> lIter = leftmap.entrySet().iterator();
+ Iterator<Map.Entry<String, ColumnInfo>> rIter = rightmap.entrySet().iterator();
+ while (lIter.hasNext()) {
+ Map.Entry<String, ColumnInfo> lEntry = lIter.next();
+ Map.Entry<String, ColumnInfo> rEntry = rIter.next();
ColumnInfo lInfo = lEntry.getValue();
- ColumnInfo rInfo = rightmap.get(field);
- if (rInfo == null) {
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(tabref,
- "Schema of both sides of union should match. " + rightalias
- + " does not have the field " + field));
- }
- if (lInfo == null) {
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(tabref,
- "Schema of both sides of union should match. " + leftalias
- + " does not have the field " + field));
- }
- if (!lInfo.getInternalName().equals(rInfo.getInternalName())) {
- throw new CalciteSemanticException(SemanticAnalyzer.generateErrorMessage(
- tabref,
- "Schema of both sides of union should match: field " + field + ":"
- + " appears on the left side of the UNION at column position: "
- + SemanticAnalyzer.getPositionFromInternalName(lInfo.getInternalName())
- + ", and on the right side of the UNION at column position: "
- + SemanticAnalyzer.getPositionFromInternalName(rInfo.getInternalName())
- + ". Column positions should match for a UNION"));
- }
- // try widening coversion, otherwise fail union
+ ColumnInfo rInfo = rEntry.getValue();
+
+ String field = lEntry.getKey();
+ // try widening conversion, otherwise fail union
TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(),
rInfo.getType());
if (commonTypeInfo == null) {
- throw new CalciteSemanticException(SemanticAnalyzer.generateErrorMessage(tabref,
- "Schema of both sides of union should match: Column " + field + " is of type "
- + lInfo.getType().getTypeName() + " on first table and type "
- + rInfo.getType().getTypeName() + " on second table"));
+ throw new SemanticException(generateErrorMessage(tabref,
+ "Schema of both sides of union should match: Column " + field
+ + " is of type " + lInfo.getType().getTypeName()
+ + " on first table and type " + rInfo.getType().getTypeName()
+ + " on second table"));
}
- }
-
- // 3. construct Union Output RR using original left & right Input
- RowResolver unionoutRR = new RowResolver();
- for (Map.Entry<String, ColumnInfo> lEntry : leftmap.entrySet()) {
- String field = lEntry.getKey();
- ColumnInfo lInfo = lEntry.getValue();
- ColumnInfo rInfo = rightmap.get(field);
ColumnInfo unionColInfo = new ColumnInfo(lInfo);
- unionColInfo.setTabAlias(unionalias);
unionColInfo.setType(FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(),
rInfo.getType()));
unionoutRR.put(unionalias, field, unionColInfo);
@@ -2637,6 +2625,7 @@ public class CalcitePlanner extends Sema
LOG.debug("Created Plan for Query Block " + qb.getId());
}
+ setQB(qb);
return srcRel;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Feb 17 06:49:27 2015
@@ -3038,7 +3038,8 @@ public class DDLSemanticAnalyzer extends
for (Entry<String, String> e : partSpec.entrySet()) {
for (String s : reservedPartitionValues) {
- if (e.getValue().contains(s)) {
+ String value = e.getValue();
+ if (value != null && value.contains(s)) {
throw new SemanticException(ErrorMsg.RESERVED_PART_VAL.getMsg(
"(User value: " + e.getValue() + " Reserved substring: " + s + ")"));
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java Tue Feb 17 06:49:27 2015
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ResourceType;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.FunctionUtils;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hive.ql.hooks.E
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
+import org.apache.hadoop.hive.ql.plan.ReloadFunctionDesc;
import org.apache.hadoop.hive.ql.plan.DropFunctionDesc;
import org.apache.hadoop.hive.ql.plan.FunctionWork;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
@@ -53,11 +55,12 @@ public class FunctionSemanticAnalyzer ex
@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
- if (ast.getToken().getType() == HiveParser.TOK_CREATEFUNCTION) {
+ if (ast.getType() == HiveParser.TOK_CREATEFUNCTION) {
analyzeCreateFunction(ast);
- }
- if (ast.getToken().getType() == HiveParser.TOK_DROPFUNCTION) {
- analyzeDropFunction(ast);
+ } else if (ast.getType() == HiveParser.TOK_DROPFUNCTION) {
+ analyzeDropFunction(ast);
+ } else if (ast.getType() == HiveParser.TOK_RELOADFUNCTION) {
+ rootTasks.add(TaskFactory.get(new FunctionWork(new ReloadFunctionDesc()), conf));
}
LOG.info("analyze done");
@@ -93,13 +96,16 @@ public class FunctionSemanticAnalyzer ex
boolean throwException =
!ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT);
- if (FunctionRegistry.getFunctionInfo(functionName) == null) {
+ FunctionInfo info = FunctionRegistry.getFunctionInfo(functionName);
+ if (info == null) {
if (throwException) {
throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName));
} else {
// Fail silently
return;
}
+ } else if (info.isBuiltIn()) {
+ throw new SemanticException(ErrorMsg.DROP_NATIVE_FUNCTION.getMsg(functionName));
}
boolean isTemporaryFunction = (ast.getFirstChildWithType(HiveParser.TOK_TEMPORARY) != null);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Feb 17 06:49:27 2015
@@ -297,6 +297,7 @@ KW_REWRITE : 'REWRITE';
KW_AUTHORIZATION: 'AUTHORIZATION';
KW_CONF: 'CONF';
KW_VALUES: 'VALUES';
+KW_RELOAD: 'RELOAD';
// Operators
// NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work.
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Feb 17 06:49:27 2015
@@ -85,7 +85,8 @@ TOK_ORDERBY;
TOK_CLUSTERBY;
TOK_DISTRIBUTEBY;
TOK_SORTBY;
-TOK_UNION;
+TOK_UNIONALL;
+TOK_UNIONDISTINCT;
TOK_JOIN;
TOK_LEFTOUTERJOIN;
TOK_RIGHTOUTERJOIN;
@@ -215,6 +216,7 @@ TOK_STRINGLITERALSEQUENCE;
TOK_CHARSETLITERAL;
TOK_CREATEFUNCTION;
TOK_DROPFUNCTION;
+TOK_RELOADFUNCTION;
TOK_CREATEMACRO;
TOK_DROPMACRO;
TOK_TEMPORARY;
@@ -612,10 +614,12 @@ import java.util.HashMap;
// counter to generate unique union aliases
private int aliasCounter;
-
private String generateUnionAlias() {
return "_u" + (++aliasCounter);
}
+ private CommonTree throwSetOpException() throws RecognitionException {
+ throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.", "");
+ }
}
@rulecatch {
@@ -699,6 +703,7 @@ ddlStatement
| createIndexStatement
| dropIndexStatement
| dropFunctionStatement
+ | reloadFunctionStatement
| dropMacroStatement
| analyzeStatement
| lockStatement
@@ -1622,6 +1627,11 @@ dropFunctionStatement
-> ^(TOK_DROPFUNCTION functionIdentifier ifExists?)
;
+reloadFunctionStatement
+@init { pushMsg("reload function statement", state); }
+@after { popMsg(state); }
+ : KW_RELOAD KW_FUNCTION -> ^(TOK_RELOADFUNCTION);
+
createMacroStatement
@init { pushMsg("create macro statement", state); }
@after { popMsg(state); }
@@ -2043,7 +2053,8 @@ unionType
setOperator
@init { pushMsg("set operator", state); }
@after { popMsg(state); }
- : KW_UNION KW_ALL -> ^(TOK_UNION)
+ : KW_UNION KW_ALL -> ^(TOK_UNIONALL)
+ | KW_UNION KW_DISTINCT? -> ^(TOK_UNIONDISTINCT)
;
queryStatementExpression[boolean topLevel]
@@ -2131,41 +2142,98 @@ regularBody[boolean topLevel]
selectStatement[topLevel]
;
- selectStatement[boolean topLevel]
- : (singleSelectStatement -> singleSelectStatement)
- (u=setOperator b=singleSelectStatement
- -> ^($u {$selectStatement.tree} $b)
- )*
- -> {u != null && topLevel}? ^(TOK_QUERY
- ^(TOK_FROM
- ^(TOK_SUBQUERY
- {$selectStatement.tree}
+selectStatement[boolean topLevel]
+ :
+ (
+ s=selectClause
+ f=fromClause?
+ w=whereClause?
+ g=groupByClause?
+ h=havingClause?
+ o=orderByClause?
+ c=clusterByClause?
+ d=distributeByClause?
+ sort=sortByClause?
+ win=window_clause?
+ l=limitClause?
+ -> ^(TOK_QUERY $f? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ $s $w? $g? $h? $o? $c?
+ $d? $sort? $win? $l?))
+ )
+ (set=setOpSelectStatement[$selectStatement.tree, topLevel])?
+ -> {set == null}?
+ {$selectStatement.tree}
+ -> {o==null && c==null && d==null && sort==null && l==null}?
+ {$set.tree}
+ -> {throwSetOpException()}
+ ;
+
+setOpSelectStatement[CommonTree t, boolean topLevel]
+ :
+ (u=setOperator b=simpleSelectStatement
+ -> {$setOpSelectStatement.tree != null && u.tree.getType()==HiveParser.TOK_UNIONDISTINCT}?
+ ^(TOK_QUERY
+ ^(TOK_FROM
+ ^(TOK_SUBQUERY
+ ^(TOK_UNIONALL {$setOpSelectStatement.tree} $b)
{adaptor.create(Identifier, generateUnionAlias())}
)
)
- ^(TOK_INSERT
+ ^(TOK_INSERT
+ ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ ^(TOK_SELECTDI ^(TOK_SELEXPR TOK_ALLCOLREF))
+ )
+ )
+ -> {$setOpSelectStatement.tree != null && u.tree.getType()!=HiveParser.TOK_UNIONDISTINCT}?
+ ^(TOK_UNIONALL {$setOpSelectStatement.tree} $b)
+ -> {$setOpSelectStatement.tree == null && u.tree.getType()==HiveParser.TOK_UNIONDISTINCT}?
+ ^(TOK_QUERY
+ ^(TOK_FROM
+ ^(TOK_SUBQUERY
+ ^(TOK_UNIONALL {$t} $b)
+ {adaptor.create(Identifier, generateUnionAlias())}
+ )
+ )
+ ^(TOK_INSERT
^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
- ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+ ^(TOK_SELECTDI ^(TOK_SELEXPR TOK_ALLCOLREF))
+ )
+ )
+ -> ^(TOK_UNIONALL {$t} $b)
+ )+
+ o=orderByClause?
+ c=clusterByClause?
+ d=distributeByClause?
+ sort=sortByClause?
+ win=window_clause?
+ l=limitClause?
+ -> {o==null && c==null && d==null && sort==null && win==null && l==null && !topLevel}?
+ {$setOpSelectStatement.tree}
+ -> ^(TOK_QUERY
+ ^(TOK_FROM
+ ^(TOK_SUBQUERY
+ {$setOpSelectStatement.tree}
+ {adaptor.create(Identifier, generateUnionAlias())}
+ )
)
- )
- -> {$selectStatement.tree}
- ;
+ ^(TOK_INSERT
+ ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+ $o? $c? $d? $sort? $win? $l?
+ )
+ )
+ ;
-singleSelectStatement
+simpleSelectStatement
:
selectClause
fromClause?
whereClause?
groupByClause?
havingClause?
- orderByClause?
- clusterByClause?
- distributeByClause?
- sortByClause?
window_clause?
- limitClause? -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
- selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
- distributeByClause? sortByClause? window_clause? limitClause?))
+ -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ selectClause whereClause? groupByClause? havingClause? window_clause?))
;
selectStatementWithCTE
@@ -2216,8 +2284,8 @@ insertClause
@after { popMsg(state); }
:
KW_INSERT KW_OVERWRITE destination ifNotExists? -> ^(TOK_DESTINATION destination ifNotExists?)
- | KW_INSERT KW_INTO KW_TABLE? tableOrPartition
- -> ^(TOK_INSERT_INTO tableOrPartition)
+ | KW_INSERT KW_INTO KW_TABLE? tableOrPartition (LPAREN targetCols=columnNameList RPAREN)?
+ -> ^(TOK_INSERT_INTO tableOrPartition $targetCols?)
;
destination
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Feb 17 06:49:27 2015
@@ -572,5 +572,5 @@ principalIdentifier
nonReserved
:
- KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN
ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER
+ KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS |
KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_
BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER | KW_RELOAD
;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java?rev=1660293&r1=1660292&r2=1660293&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java Tue Feb 17 06:49:27 2015
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.optimizer.IndexUtils;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -69,7 +70,7 @@ public class IndexUpdater {
for (LoadTableDesc ltd : loadTableWork) {
TableDesc td = ltd.getTable();
Table srcTable = hive.getTable(td.getTableName());
- List<Index> tblIndexes = srcTable.getAllIndexes((short)-1);
+ List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1);
Map<String, String> partSpec = ltd.getPartitionSpec();
if (partSpec == null || partSpec.size() == 0) {
//unpartitioned table, update whole index