You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/08/26 23:42:21 UTC
svn commit: r1517707 [9/17] - in /hive/branches/tez: ./
beeline/src/java/org/apache/hive/beeline/ bin/ bin/ext/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/java/org/apache/hadoop/hive/co...
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Mon Aug 26 21:42:12 2013
@@ -19,23 +19,24 @@
package org.apache.hadoop.hive.ql.optimizer.ppr;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -55,8 +56,6 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.thrift.TException;
/**
* The transformation step that does partition pruning.
@@ -84,8 +83,6 @@ public class PartitionPruner implements
/* Move logic to PrunerUtils.walkOperatorTree() so that it can be reused. */
PrunerUtils.walkOperatorTree(pctx, opWalkerCtx, OpProcFactory.getFilterProc(),
OpProcFactory.getDefaultProc());
- pctx.setHasNonPartCols(opWalkerCtx.getHasNonPartCols());
-
return pctx;
}
@@ -130,6 +127,16 @@ public class PartitionPruner implements
}
/**
+ * Get the partition list for the TS operator that satisfies the partition pruner
+ * condition.
+ */
+ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx,
+ String alias) throws HiveException {
+ return prune(parseCtx.getTopToTable().get(ts), parseCtx.getOpToPartPruner().get(ts),
+ parseCtx.getConf(), alias, parseCtx.getPrunedPartitions());
+ }
+
+ /**
* Get the partition list for the table that satisfies the partition pruner
* condition.
*
@@ -141,13 +148,15 @@ public class PartitionPruner implements
* for checking whether "strict" mode is on.
* @param alias
* for generating error message only.
+ * @param prunedPartitionsMap
+ * cached result for the table
* @return the partition list for the table that satisfies the partition
* pruner condition.
* @throws HiveException
*/
- public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
- HiveConf conf, String alias,
- Map<String, PrunedPartitionList> prunedPartitionsMap) throws HiveException {
+ private static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
+ HiveConf conf, String alias, Map<String, PrunedPartitionList> prunedPartitionsMap)
+ throws HiveException {
LOG.trace("Started pruning partiton");
LOG.trace("dbname = " + tab.getDbName());
LOG.trace("tabname = " + tab.getTableName());
@@ -163,82 +172,15 @@ public class PartitionPruner implements
return ret;
}
- LinkedHashSet<Partition> true_parts = new LinkedHashSet<Partition>();
- LinkedHashSet<Partition> unkn_parts = new LinkedHashSet<Partition>();
- LinkedHashSet<Partition> denied_parts = new LinkedHashSet<Partition>();
-
- try {
- StructObjectInspector rowObjectInspector = (StructObjectInspector) tab
- .getDeserializer().getObjectInspector();
- Object[] rowWithPart = new Object[2];
-
- if (tab.isPartitioned()) {
- // If the "strict" mode is on, we have to provide partition pruner for
- // each table.
- if ("strict".equalsIgnoreCase(HiveConf.getVar(conf,
- HiveConf.ConfVars.HIVEMAPREDMODE))) {
- if (!hasColumnExpr(prunerExpr)) {
- throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE
- .getMsg("for Alias \"" + alias + "\" Table \""
- + tab.getTableName() + "\""));
- }
- }
-
- if (prunerExpr == null) {
- // This can happen when hive.mapred.mode=nonstrict and there is no predicates at all
- // Add all partitions to the unknown_parts so that a MR job is generated.
- true_parts.addAll(Hive.get().getPartitions(tab));
- } else {
- // remove non-partition columns
- ExprNodeDesc compactExpr = prunerExpr.clone();
- compactExpr = compactExpr(compactExpr);
- LOG.debug("Filter w/ compacting: " +
- ((compactExpr != null) ? compactExpr.getExprString(): "null") +
- "; filter w/o compacting: " +
- ((prunerExpr != null) ? prunerExpr.getExprString(): "null"));
- if (compactExpr == null) {
- // This could happen when hive.mapred.mode=nonstrict and all the predicates
- // are on non-partition columns.
- unkn_parts.addAll(Hive.get().getPartitions(tab));
- } else {
- String message = Utilities.checkJDOPushDown(tab, compactExpr, null);
- if (message == null) {
- String filter = compactExpr.getExprString();
- String oldFilter = prunerExpr.getExprString();
-
- if (filter.equals(oldFilter)) {
- // pruneExpr contains only partition columns
- pruneByPushDown(tab, true_parts, filter);
- } else {
- // pruneExpr contains non-partition columns
- pruneByPushDown(tab, unkn_parts, filter);
- }
- } else {
- LOG.info(ErrorMsg.INVALID_JDO_FILTER_EXPRESSION.getMsg("by condition '"
- + message + "'"));
- pruneBySequentialScan(tab, true_parts, unkn_parts, denied_parts,
- prunerExpr, rowObjectInspector, conf);
- }
- }
- }
- LOG.debug("tabname = " + tab.getTableName() + " is partitioned");
- } else {
- true_parts.addAll(Hive.get().getPartitions(tab));
- }
- } catch (HiveException e) {
- throw e;
- } catch (Exception e) {
- throw new HiveException(e);
- }
-
- // Now return the set of partitions
- ret = new PrunedPartitionList(tab, true_parts, unkn_parts, denied_parts);
+ ret = getPartitionsFromServer(tab, prunerExpr, conf, alias);
prunedPartitionsMap.put(key, ret);
return ret;
}
/**
- * Taking a partition pruning expression, remove the null operands.
+ * Taking a partition pruning expression, remove the null operands and non-partition columns.
+ * The reason why there are null operands is ExprProcFactory classes, for example
+ * PPRColumnExprProcessor.
* @param expr original partition pruning expression.
* @return partition pruning expression that only contains partition columns.
*/
@@ -251,19 +193,21 @@ public class PartitionPruner implements
}
} else if (expr instanceof ExprNodeGenericFuncDesc) {
GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
- if (udf instanceof GenericUDFOPAnd ||
- udf instanceof GenericUDFOPOr) {
- List<ExprNodeDesc> children = ((ExprNodeGenericFuncDesc)expr).getChildren();
+ boolean isAnd = udf instanceof GenericUDFOPAnd;
+ if (isAnd || udf instanceof GenericUDFOPOr) {
+ List<ExprNodeDesc> children = expr.getChildren();
ExprNodeDesc left = children.get(0);
children.set(0, compactExpr(left));
ExprNodeDesc right = children.get(1);
children.set(1, compactExpr(right));
+ // Note that one does not simply compact (not-null or null) to not-null.
+ // Only if we have an "and" is it valid to send one side to metastore.
if (children.get(0) == null && children.get(1) == null) {
return null;
} else if (children.get(0) == null) {
- return children.get(1);
+ return isAnd ? children.get(1) : null;
} else if (children.get(1) == null) {
- return children.get(0);
+ return isAnd ? children.get(0) : null;
}
}
return expr;
@@ -272,111 +216,169 @@ public class PartitionPruner implements
}
/**
- * Pruning partition using JDO filtering.
- * @param tab the table containing the partitions.
- * @param true_parts the resulting partitions.
- * @param filter the SQL predicate that involves only partition columns
- * @throws HiveException
- * @throws MetaException
- * @throws NoSuchObjectException
- * @throws TException
+ * See compactExpr. Some things in the expr are replaced with nulls for pruner, however
+ * the virtual columns are not removed (ExprNodeColumnDesc cannot tell them apart from
+ * partition columns), so we do it here.
+ * The expression is only used to prune by partition name, so we have no business with VCs.
+ * @param expr original partition pruning expression.
+ * @param partCols list of partition columns for the table.
+ * @return partition pruning expression that only contains partition columns from the list.
*/
- static private void pruneByPushDown(Table tab, Set<Partition> true_parts, String filter)
- throws HiveException, MetaException, NoSuchObjectException, TException {
- Hive db = Hive.get();
- List<Partition> parts = db.getPartitionsByFilter(tab, filter);
- true_parts.addAll(parts);
- return;
+ static private ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List<String> partCols) {
+ if (expr instanceof ExprNodeColumnDesc
+ && !partCols.contains(((ExprNodeColumnDesc) expr).getColumn())) {
+ // Column doesn't appear to be a partition column for the table.
+ return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
+ }
+ if (expr instanceof ExprNodeGenericFuncDesc) {
+ List<ExprNodeDesc> children = expr.getChildren();
+ for (int i = 0; i < children.size(); ++i) {
+ children.set(i, removeNonPartCols(children.get(i), partCols));
+ }
+ }
+ return expr;
+ }
+
+ private static PrunedPartitionList getPartitionsFromServer(Table tab,
+ ExprNodeDesc prunerExpr, HiveConf conf, String alias) throws HiveException {
+ try {
+ if (!tab.isPartitioned()) {
+ // If the table is not partitioned, return everything.
+ return new PrunedPartitionList(
+ tab, new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), false);
+ }
+ LOG.debug("tabname = " + tab.getTableName() + " is partitioned");
+
+ if ("strict".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE))
+ && !hasColumnExpr(prunerExpr)) {
+ // If the "strict" mode is on, we have to provide partition pruner for each table.
+ throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE
+ .getMsg("for Alias \"" + alias + "\" Table \"" + tab.getTableName() + "\""));
+ }
+
+ if (prunerExpr == null) {
+ // This can happen when hive.mapred.mode=nonstrict and there is no predicates at all.
+ return new PrunedPartitionList(tab,
+ new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), false);
+ }
+
+ // Remove virtual columns. See javadoc for details.
+ prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab));
+ // Remove all unknown parts e.g. non-partition columns. See javadoc for details.
+ ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
+ String oldFilter = prunerExpr.getExprString();
+ if (compactExpr == null) {
+ // This could happen when hive.mapred.mode=nonstrict and all the predicates
+ // are on non-partition columns.
+ LOG.debug("Filter " + oldFilter + " was null after compacting");
+ return new PrunedPartitionList(
+ tab, new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), true);
+ }
+
+ Set<Partition> partitions = new LinkedHashSet<Partition>();
+ boolean hasUnknownPartitions = false;
+ String message = Utilities.checkJDOPushDown(tab, compactExpr, null);
+ if (message != null) {
+ LOG.info(ErrorMsg.INVALID_JDO_FILTER_EXPRESSION.getMsg("by condition '"
+ + message + "'"));
+ hasUnknownPartitions = pruneBySequentialScan(tab, partitions, prunerExpr, conf);
+ } else {
+ String filter = compactExpr.getExprString();
+ LOG.debug("Filter w/ compacting: " + filter +"; filter w/o compacting: " + oldFilter);
+ hasUnknownPartitions = !filter.equals(oldFilter);
+ partitions.addAll(Hive.get().getPartitionsByFilter(tab, filter));
+ }
+ return new PrunedPartitionList(tab, partitions, hasUnknownPartitions);
+ } catch (HiveException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
}
/**
* Pruning partition by getting the partition names first and pruning using Hive expression
* evaluator.
* @param tab the table containing the partitions.
- * @param true_parts the resulting partitions if the partition pruning expression only contains
- * partition columns.
- * @param unkn_parts the resulting partitions if the partition pruning expression that only contains
- * non-partition columns.
- * @param denied_parts pruned out partitions.
+ * @param partitions the resulting partitions.
* @param prunerExpr the SQL predicate that involves partition columns.
- * @param rowObjectInspector object inspector used by the evaluator
* @param conf Hive Configuration object, can not be NULL.
- * @throws Exception
+ * @return true iff the partition pruning expression contains non-partition columns.
*/
- static private void pruneBySequentialScan(Table tab, Set<Partition> true_parts, Set<Partition> unkn_parts,
- Set<Partition> denied_parts, ExprNodeDesc prunerExpr, StructObjectInspector rowObjectInspector, HiveConf conf)
- throws Exception {
+ static private boolean pruneBySequentialScan(Table tab, Set<Partition> partitions,
+ ExprNodeDesc prunerExpr, HiveConf conf) throws Exception {
+ PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ perfLogger.PerfLogBegin(LOG, PerfLogger.PRUNE_LISTING);
- List<String> trueNames = null;
- List<String> unknNames = null;
+ List<String> partNames = Hive.get().getPartitionNames(
+ tab.getDbName(), tab.getTableName(), (short) -1);
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME);
+ List<String> partCols = extractPartColNames(tab);
- perfLogger.PerfLogBegin(LOG, PerfLogger.PRUNE_LISTING);
+ boolean hasUnknownPartitions = prunePartitionNames(
+ partCols, prunerExpr, defaultPartitionName, partNames);
+ perfLogger.PerfLogEnd(LOG, PerfLogger.PRUNE_LISTING);
- List<String> partNames = Hive.get().getPartitionNames(tab.getDbName(),
- tab.getTableName(), (short) -1);
+ perfLogger.PerfLogBegin(LOG, PerfLogger.PARTITION_RETRIEVING);
+ if (!partNames.isEmpty()) {
+ partitions.addAll(Hive.get().getPartitionsByNames(tab, partNames));
+ }
+ perfLogger.PerfLogEnd(LOG, PerfLogger.PARTITION_RETRIEVING);
+ return hasUnknownPartitions;
+ }
+ private static List<String> extractPartColNames(Table tab) {
List<FieldSchema> pCols = tab.getPartCols();
List<String> partCols = new ArrayList<String>(pCols.size());
- List<String> values = new ArrayList<String>(pCols.size());
- Object[] objectWithPart = new Object[2];
- String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME);
-
for (FieldSchema pCol : pCols) {
partCols.add(pCol.getName());
}
+ return partCols;
+ }
- Map<PrimitiveObjectInspector, ExprNodeEvaluator> handle = PartExprEvalUtils.prepareExpr(
- prunerExpr, partCols, rowObjectInspector);
-
- for (String partName : partNames) {
-
- // Set all the variables here
- LinkedHashMap<String, String> partSpec = Warehouse
- .makeSpecFromName(partName);
-
+ /**
+ * Prunes partition names to see if they match the prune expression.
+ * @param columnNames name of partition columns
+ * @param prunerExpr The expression to match.
+ * @param defaultPartitionName name of default partition
+ * @param partNames Partition names to filter. The list is modified in place.
+ * @return Whether the list has any partitions for which the expression may or may not match.
+ */
+ public static boolean prunePartitionNames(List<String> columnNames, ExprNodeDesc prunerExpr,
+ String defaultPartitionName, List<String> partNames) throws HiveException, MetaException {
+ // Prepare the expression to filter on the columns.
+ ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle =
+ PartExprEvalUtils.prepareExpr(prunerExpr, columnNames);
+
+ // Filter the name list.
+ List<String> values = new ArrayList<String>(columnNames.size());
+ boolean hasUnknownPartitions = false;
+ Iterator<String> partIter = partNames.iterator();
+ while (partIter.hasNext()) {
+ String partName = partIter.next();
+ LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
values.clear();
- for (Map.Entry<String, String> kv: partSpec.entrySet()) {
- values.add(kv.getValue());
- }
- objectWithPart[1] = values;
-
- // evaluate the expression tree
- Boolean r = (Boolean) PartExprEvalUtils.evaluateExprOnPart(handle, objectWithPart);
+ values.addAll(partSpec.values());
- if (r == null) {
+ // Evaluate the expression tree.
+ Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, values);
+ boolean isUnknown = (isNeeded == null);
+ if (!isUnknown && !isNeeded) {
+ partIter.remove();
+ continue;
+ }
+ if (isUnknown && values.contains(defaultPartitionName)) {
// Reject default partitions if we couldn't determine whether we should include it or not.
// Note that predicate would only contains partition column parts of original predicate.
- if (values.contains(defaultPartitionName)) {
- LOG.debug("skipping default/bad partition: " + partName);
- }else {
- if (unknNames == null) {
- unknNames = new LinkedList<String>();
- }
- unknNames.add(partName);
- LOG.debug("retained unknown partition: " + partName);
- }
- } else if (Boolean.TRUE.equals(r)) {
- if (trueNames == null) {
- trueNames = new LinkedList<String>();
- }
- trueNames.add(partName);
- LOG.debug("retained partition: " + partName);
+ LOG.debug("skipping default/bad partition: " + partName);
+ partIter.remove();
+ continue;
}
+ hasUnknownPartitions |= isUnknown;
+ LOG.debug("retained " + (isUnknown ? "unknown " : "") + "partition: " + partName);
}
- perfLogger.PerfLogEnd(LOG, PerfLogger.PRUNE_LISTING);
-
- perfLogger.PerfLogBegin(LOG, PerfLogger.PARTITION_RETRIEVING);
- if (trueNames != null) {
- List<Partition> parts = Hive.get().getPartitionsByNames(tab, trueNames);
- true_parts.addAll(parts);
- }
- if (unknNames != null) {
- List<Partition> parts = Hive.get().getPartitionsByNames(tab, unknNames);
- unkn_parts.addAll(parts);
- }
- perfLogger.PerfLogEnd(LOG, PerfLogger.PARTITION_RETRIEVING);
+ return hasUnknownPartitions;
}
/**
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Aug 26 21:42:12 2013
@@ -986,13 +986,26 @@ public abstract class BaseSemanticAnalyz
ListBucketingCtx lbCtx = new ListBucketingCtx();
lbCtx.setSkewedColNames(skewedColNames);
lbCtx.setSkewedColValues(skewedValues);
- lbCtx.setLbLocationMap(skewedColValueLocationMaps);
+ lbCtx.setLbLocationMap(convertSkewedValueListToSimpleList(skewedColValueLocationMaps));
lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories);
lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY);
lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME);
return lbCtx;
}
+ // This is done to avoid the need of sending metastore jars to task nodes.
+ private Map<List<String>, String> convertSkewedValueListToSimpleList(
+ Map<SkewedValueList, String> skewedColValueLocationMaps) {
+ if (skewedColValueLocationMaps == null) {
+ return null;
+ }
+ Map<List<String>, String> converted = new HashMap<List<String>, String>();
+ for (Map.Entry<SkewedValueList, String> entry : skewedColValueLocationMaps.entrySet()) {
+ converted.put(entry.getKey().getSkewedValueList(), entry.getValue());
+ }
+ return converted;
+ }
+
/**
* Given a ASTNode, return list of values.
*
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Mon Aug 26 21:42:12 2013
@@ -125,11 +125,11 @@ lateralView
@init {gParent.msgs.push("lateral view"); }
@after {gParent.msgs.pop(); }
:
- KW_LATERAL KW_VIEW KW_OUTER function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
|
- KW_LATERAL KW_VIEW function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
;
tableAlias
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Mon Aug 26 21:42:12 2013
@@ -173,8 +173,9 @@ function
(star=STAR)
| (dist=KW_DISTINCT)? (selectExpression (COMMA selectExpression)*)?
)
- RPAREN -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName)
- -> {$dist == null}? ^(TOK_FUNCTION functionName (selectExpression+)?)
+ RPAREN (KW_OVER ws=window_specification)?
+ -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName $ws?)
+ -> {$dist == null}? ^(TOK_FUNCTION functionName (selectExpression+)? $ws?)
-> ^(TOK_FUNCTIONDI functionName (selectExpression+)?)
;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Mon Aug 26 21:42:12 2013
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.serde.serd
*/
public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
+ public static final String METADATA_NAME="_metadata";
+
public ImportSemanticAnalyzer(HiveConf conf) throws SemanticException {
super(conf);
}
@@ -89,7 +91,7 @@ public class ImportSemanticAnalyzer exte
Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(),
fromURI.getPath());
try {
- Path metadataPath = new Path(fromPath, "_metadata");
+ Path metadataPath = new Path(fromPath, METADATA_NAME);
Map.Entry<org.apache.hadoop.hive.metastore.api.Table,
List<Partition>> rv = EximUtil.readMetaData(fs, metadataPath);
dbname = db.getCurrentDatabase();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java Mon Aug 26 21:42:12 2013
@@ -38,14 +38,13 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.events.PreEventContext;
import org.apache.hadoop.hive.metastore.events.PreEventContext.PreEventType;
import org.apache.hadoop.hive.ql.session.SessionState;
-
+import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
/**
* This class listens for drop events and, if set, exports the table's metadata as JSON to the trash
* of the user performing the drop
*/
public class MetaDataExportListener extends MetaStorePreEventListener {
public static final Log LOG = LogFactory.getLog(MetaDataExportListener.class);
- private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
/** Configure the export listener */
public MetaDataExportListener(Configuration config) {
@@ -65,6 +64,7 @@ public class MetaDataExportListener exte
Path tblPath = new Path(tbl.getSd().getLocation());
fs = wh.getFs(tblPath);
Date now = new Date();
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
String dateString = sdf.format(now);
String exportPathString = hiveconf.getVar(HiveConf.ConfVars.METADATA_EXPORT_LOCATION);
boolean moveMetadataToTrash = hiveconf
@@ -83,7 +83,7 @@ public class MetaDataExportListener exte
} catch (IOException e) {
throw new MetaException(e.getMessage());
}
- Path outFile = new Path(metaPath, name + ".metadata");
+ Path outFile = new Path(metaPath, name + ImportSemanticAnalyzer.METADATA_NAME);
try {
SessionState.getConsole().printInfo("Beginning metadata export");
EximUtil.createExportDump(fs, outFile, mTbl, null);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Mon Aug 26 21:42:12 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.ErrorMs
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
@@ -131,6 +132,7 @@ public class PTFTranslator {
init(semAly, hCfg, inputRR, unparseT);
ptfInvocation = qSpec;
ptfDesc = new PTFDesc();
+ ptfDesc.setCfg(hCfg);
ptfDesc.setLlInfo(llInfo);
translatePTFChain();
return ptfDesc;
@@ -143,6 +145,7 @@ public class PTFTranslator {
init(semAly, hCfg, inputRR, unparseT);
windowingSpec = wdwSpec;
ptfDesc = new PTFDesc();
+ ptfDesc.setCfg(hCfg);
ptfDesc.setLlInfo(llInfo);
WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
ptfDesc.setFuncDef(wdwTFnDef);
@@ -693,7 +696,8 @@ public class PTFTranslator {
try {
serde = PTFTranslator.createLazyBinarySerDe(hCfg, OI, serdePropsMap);
- shp.setOI((StructObjectInspector) serde.getObjectInspector());
+ StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serde, OI);
+ shp.setOI(outOI);
} catch (SerDeException se) {
throw new SemanticException(se);
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java Mon Aug 26 21:42:12 2013
@@ -100,15 +100,6 @@ public class ParseContext {
*/
private LineageInfo lInfo;
- // is set to true if the expression only contains partitioning columns and not
- // any other column reference.
- // This is used to optimize select * from table where ... scenario, when the
- // where condition only references
- // partitioning columns - the partitions are identified and streamed directly
- // to the client without requiring
- // a map-reduce job
- private boolean hasNonPartCols;
-
private GlobalLimitCtx globalLimitCtx;
private HashSet<ReadEntity> semanticInputs;
@@ -206,7 +197,6 @@ public class ParseContext {
this.destTableId = destTableId;
this.uCtx = uCtx;
this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer;
- hasNonPartCols = false;
this.groupOpToInputTables = groupOpToInputTables;
this.prunedPartitions = prunedPartitions;
this.opToSamplePruner = opToSamplePruner;
@@ -515,22 +505,6 @@ public class ParseContext {
}
/**
- * Sets the hasNonPartCols flag.
- *
- * @param val
- */
- public void setHasNonPartCols(boolean val) {
- hasNonPartCols = val;
- }
-
- /**
- * Gets the value of the hasNonPartCols flag.
- */
- public boolean getHasNonPartCols() {
- return hasNonPartCols;
- }
-
- /**
* @return the opToSamplePruner
*/
public HashMap<TableScanOperator, sampleDesc> getOpToSamplePruner() {
@@ -640,8 +614,7 @@ public class ParseContext {
throws HiveException {
PrunedPartitionList partsList = opToPartList.get(ts);
if (partsList == null) {
- partsList = PartitionPruner.prune(topToTable.get(ts),
- opToPartPruner.get(ts), conf, alias, prunedPartitions);
+ partsList = PartitionPruner.prune(ts, this, alias);
opToPartList.put(ts, partsList);
}
return partsList;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java Mon Aug 26 21:42:12 2013
@@ -30,30 +30,19 @@ import org.apache.hadoop.hive.ql.metadat
*/
public class PrunedPartitionList {
- // source table
+ /** Source table. */
private final Table source;
- // confirmed partitions - satisfy the partition criteria
- private Set<Partition> confirmedPartns;
+ /** Partitions that either satisfy the partition criteria, or may satisfy it. */
+ private Set<Partition> partitions;
- // unknown partitions - may/may not satisfy the partition criteria
- private Set<Partition> unknownPartns;
+ /** Whether there are partitions in the list that may or may not satisfy the criteria. */
+ private boolean hasUnknowns;
- // denied partitions - do not satisfy the partition criteria
- private final Set<Partition> deniedPartns;
-
- /**
- * @param confirmedPartns
- * confirmed partitions
- * @param unknownPartns
- * unknown partitions
- */
- public PrunedPartitionList(Table source, Set<Partition> confirmedPartns,
- Set<Partition> unknownPartns, Set<Partition> deniedPartns) {
+ public PrunedPartitionList(Table source, Set<Partition> partitions, boolean hasUnknowns) {
this.source = source;
- this.confirmedPartns = confirmedPartns;
- this.unknownPartns = unknownPartns;
- this.deniedPartns = deniedPartns;
+ this.partitions = partitions;
+ this.hasUnknowns = hasUnknowns;
}
public Table getSourceTable() {
@@ -61,59 +50,24 @@ public class PrunedPartitionList {
}
/**
- * get confirmed partitions.
- *
- * @return confirmedPartns confirmed paritions
- */
- public Set<Partition> getConfirmedPartns() {
- return confirmedPartns;
- }
-
- /**
- * get unknown partitions.
- *
- * @return unknownPartns unknown paritions
+ * @return partitions
*/
- public Set<Partition> getUnknownPartns() {
- return unknownPartns;
+ public Set<Partition> getPartitions() {
+ return partitions;
}
- /**
- * get denied partitions.
- *
- * @return deniedPartns denied paritions
- */
- public Set<Partition> getDeniedPartns() {
- return deniedPartns;
- }
/**
- * return all not-denied(confirmed + unknown) partitions.
+ * @return all partitions.
*/
public List<Partition> getNotDeniedPartns() {
- List<Partition> partitions = new ArrayList<Partition>();
- partitions.addAll(confirmedPartns);
- partitions.addAll(unknownPartns);
- return partitions;
- }
-
- /**
- * set confirmed partitions.
- *
- * @param confirmedPartns
- * confirmed paritions
- */
- public void setConfirmedPartns(Set<Partition> confirmedPartns) {
- this.confirmedPartns = confirmedPartns;
+ return new ArrayList<Partition>(partitions);
}
/**
- * set unknown partitions.
- *
- * @param unknownPartns
- * unknown partitions
+ * @return Whether there are unknown partitions in {@link #getPartitions()} result.
*/
- public void setUnknownPartns(Set<Partition> unknownPartns) {
- this.unknownPartns = unknownPartns;
+ public boolean hasUnknownPartitions() {
+ return hasUnknowns;
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g Mon Aug 26 21:42:12 2013
@@ -125,9 +125,9 @@ selectItem
@init { gParent.msgs.push("selection target"); }
@after { gParent.msgs.pop(); }
:
- ( selectExpression (KW_OVER ws=window_specification )?
+ ( selectExpression
((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?
- ) -> ^(TOK_SELEXPR selectExpression identifier* $ws?)
+ ) -> ^(TOK_SELEXPR selectExpression identifier*)
;
trfmClause
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon Aug 26 21:42:12 2013
@@ -71,7 +71,6 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
@@ -360,29 +359,39 @@ public class SemanticAnalyzer extends Ba
}
private LinkedHashMap<String, ASTNode> doPhase1GetAggregationsFromSelect(
- ASTNode selExpr, QB qb, String dest) {
+ ASTNode selExpr, QB qb, String dest) throws SemanticException {
// Iterate over the selects search for aggregation Trees.
// Use String as keys to eliminate duplicate trees.
LinkedHashMap<String, ASTNode> aggregationTrees = new LinkedHashMap<String, ASTNode>();
+ List<ASTNode> wdwFns = new ArrayList<ASTNode>();
for (int i = 0; i < selExpr.getChildCount(); ++i) {
- ASTNode sel = (ASTNode) selExpr.getChild(i);
- doPhase1GetAllAggregations((ASTNode) sel.getChild(0), aggregationTrees);
+ ASTNode function = (ASTNode) selExpr.getChild(i).getChild(0);
+ doPhase1GetAllAggregations((ASTNode) function, aggregationTrees, wdwFns);
}
- /*
- * remove any aggregation to be handled by Windowing.
- */
- if ( queryProperties.hasWindowing() && qb.getWindowingSpec(dest) != null ) {
- HashMap<String, ASTNode> aliasToWdwExprs = qb.getParseInfo().getWindowingExprsForClause(dest);
- LinkedHashMap<String, ASTNode> aggTreesMinusWindowing = new LinkedHashMap<String, ASTNode>();
- for(Map.Entry<String,ASTNode> entry : aggregationTrees.entrySet()) {
- if ( !aliasToWdwExprs.containsKey(entry.getKey())) {
- aggTreesMinusWindowing.put(entry.getKey(), entry.getValue());
- }
+ // window based aggregations are handled differently
+ for (ASTNode wdwFn : wdwFns) {
+ WindowingSpec spec = qb.getWindowingSpec(dest);
+ if(spec == null) {
+ queryProperties.setHasWindowing(true);
+ spec = new WindowingSpec();
+ qb.addDestToWindowingSpec(dest, spec);
+ }
+ HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
+ int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
+ WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn,
+ (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1));
+ // If this is a duplicate invocation of a function; don't add to WindowingSpec.
+ if ( wExprsInDest != null &&
+ wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
+ continue;
}
- aggregationTrees = aggTreesMinusWindowing;
+ wFnSpec.setAlias("_wcol" + wColIdx++);
+ spec.addWindowFunction(wFnSpec);
+ qb.getParseInfo().addWindowingExprToClause(dest, wFnSpec.getExpression());
}
+
return aggregationTrees;
}
@@ -406,18 +415,30 @@ public class SemanticAnalyzer extends Ba
* @param aggregations
* the key to the HashTable is the toStringTree() representation of
* the aggregation subtree.
+ * @throws SemanticException
*/
private void doPhase1GetAllAggregations(ASTNode expressionTree,
- HashMap<String, ASTNode> aggregations) {
+ HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns) throws SemanticException {
int exprTokenType = expressionTree.getToken().getType();
if (exprTokenType == HiveParser.TOK_FUNCTION
|| exprTokenType == HiveParser.TOK_FUNCTIONDI
|| exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
assert (expressionTree.getChildCount() != 0);
+ if (expressionTree.getChild(expressionTree.getChildCount()-1).getType()
+ == HiveParser.TOK_WINDOWSPEC) {
+ wdwFns.add(expressionTree);
+ return;
+ }
if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
String functionName = unescapeIdentifier(expressionTree.getChild(0)
.getText());
+ if(FunctionRegistry.impliesOrder(functionName)) {
+ throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
+ }
if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) {
+ if(containsLeadLagUDF(expressionTree)) {
+ throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
+ }
aggregations.put(expressionTree.toStringTree(), expressionTree);
FunctionInfo fi = FunctionRegistry.getFunctionInfo(functionName);
if (!fi.isNative()) {
@@ -430,7 +451,7 @@ public class SemanticAnalyzer extends Ba
}
for (int i = 0; i < expressionTree.getChildCount(); i++) {
doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(i),
- aggregations);
+ aggregations, wdwFns);
}
}
@@ -768,8 +789,6 @@ public class SemanticAnalyzer extends Ba
qbp.setHints((ASTNode) ast.getChild(0));
}
- handleWindowingExprsInSelectList(qb, ctx_1.dest, ast);
-
LinkedHashMap<String, ASTNode> aggregations = doPhase1GetAggregationsFromSelect(ast,
qb, ctx_1.dest);
doPhase1GetColumnAliasesFromSelect(ast, qbp);
@@ -2493,14 +2512,12 @@ public class SemanticAnalyzer extends Ba
(ASTNode) selExprList.getChild(1),
ErrorMsg.UDTF_MULTIPLE_EXPR.getMsg()));
}
- // Require an AS for UDTFs for column aliases
+
ASTNode selExpr = (ASTNode) selExprList.getChild(posn);
- if (selExpr.getChildCount() < 2) {
- throw new SemanticException(generateErrorMessage(udtfExpr,
- ErrorMsg.UDTF_REQUIRE_AS.getMsg()));
- }
+
// Get the column / table aliases from the expression. Start from 1 as
// 0 is the TOK_FUNCTION
+ // column names also can be inferred from result of UDTF
for (int i = 1; i < selExpr.getChildCount(); i++) {
ASTNode selExprChild = (ASTNode) selExpr.getChild(i);
switch (selExprChild.getType()) {
@@ -5490,9 +5507,15 @@ public class SemanticAnalyzer extends Ba
}
StructObjectInspector outputOI = genericUDTF.initialize(colOIs);
+ int numUdtfCols = outputOI.getAllStructFieldRefs().size();
+ if (colAliases.isEmpty()) {
+ // user did not specfied alias names, infer names from outputOI
+ for (StructField field : outputOI.getAllStructFieldRefs()) {
+ colAliases.add(field.getFieldName());
+ }
+ }
// Make sure that the number of column aliases in the AS clause matches
// the number of columns output by the UDTF
- int numUdtfCols = outputOI.getAllStructFieldRefs().size();
int numSuppliedAliases = colAliases.size();
if (numUdtfCols != numSuppliedAliases) {
throw new SemanticException(ErrorMsg.UDTF_ALIAS_MISMATCH
@@ -6468,7 +6491,6 @@ public class SemanticAnalyzer extends Ba
}
children[1] = alias;
joinTree.setBaseSrc(children);
- aliasToOpInfo.get(alias);
joinTree.setId(qb.getId());
joinTree.getAliasToOpInfo().put(
getModifiedAlias(qb, alias), aliasToOpInfo.get(alias));
@@ -7338,8 +7360,9 @@ public class SemanticAnalyzer extends Ba
if (limit != null) {
boolean extraMRStep = true;
- if (qb.getIsQuery() && qbp.getClusterByForClause(dest) == null
- && qbp.getSortByForClause(dest) == null) {
+ if (qbp.getOrderByForClause(dest) != null ||
+ qb.getIsQuery() && qbp.getClusterByForClause(dest) == null &&
+ qbp.getSortByForClause(dest) == null) {
extraMRStep = false;
}
@@ -7718,10 +7741,10 @@ public class SemanticAnalyzer extends Ba
TableScanDesc tsDesc = new TableScanDesc(alias, vcList);
setupStats(tsDesc, qb.getParseInfo(), tab, alias, rwsch);
- SplitSample sample = nameToSplitSample.get(alias);
+ SplitSample sample = nameToSplitSample.get(alias_id);
if (sample != null && sample.getRowCount() != null) {
tsDesc.setRowLimit(sample.getRowCount());
- nameToSplitSample.remove(alias);
+ nameToSplitSample.remove(alias_id);
}
top = putOpInsertMap(OperatorFactory.get(tsDesc,
@@ -9480,159 +9503,6 @@ public class SemanticAnalyzer extends Ba
qb.addPTFNodeToSpec(ptf, spec);
}
-//--------------------------- Windowing handling -----------------------------------
-
- /*
- * - A Select Item form is: ^(TOK_SELEXPR selectExpression Identifier* window_specification?)
- * What makes a UDAF invocation a Windowing Function invocation:
- * 1. It appears in a SelectExpr that as a WindowSpec
- * 2. It is a UDAF that implies order (FunctionRegistry.impliesOrder)
- * 3. It contains lead/lag UDF invocations in its args.
- */
- private boolean checkAndExtractWindowFunctionsInSelect(QB qb, ASTNode selectExpr, String dest)
- throws SemanticException {
-
- int childCount = selectExpr.getChildCount();
- ASTNode windowSpec = (ASTNode) selectExpr.getChild(childCount - 1);
-
- boolean hasWindowSpec = windowSpec.getType() == HiveParser.TOK_WINDOWSPEC;
-
- ArrayList<ASTNode> functions =
- extractWindowingUDAFs((ASTNode) selectExpr.getChild(0), !hasWindowSpec);
- if ( functions.size() == 0 ) {
- return false;
- }
-
- WindowingSpec spec = qb.getWindowingSpec(dest);
- if(spec == null) {
- queryProperties.setHasWindowing(true);
- spec = new WindowingSpec();
- qb.addDestToWindowingSpec(dest, spec);
- }
-
- HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
- int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
- for(ASTNode function : functions) {
- WindowFunctionSpec wFnSpec = processWindowFunction(function,
- hasWindowSpec ? windowSpec : null);
-
- /*
- * If this is a duplicate invocation of a function; don't add to WindowingSpec.
- */
- if ( wExprsInDest != null &&
- wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
- continue;
- }
- wFnSpec.setAlias("_wcol" + wColIdx++);
- spec.addWindowFunction(wFnSpec);
- qb.getParseInfo().addWindowingExprToClause(dest, wFnSpec.getExpression());
- }
- return true;
- }
-
- /*
- * return the UDAFs within the expressionTree.
- * If implyOrder is true, then only return the invocations that:
- * - are for UDAFs that implyOrder (FunctionRegistry.implyOrder)
- * - or contain a Lead/Lag UDF invocation in their arguments
- * If implyOrder is false, then return all UDAF invocations.
- */
- private ArrayList<ASTNode> extractWindowingUDAFs(ASTNode expressionTree, boolean implyOrder) {
- ArrayList<ASTNode> aggregations = new ArrayList<ASTNode>();
- extractWindowingUDAFs(expressionTree, aggregations);
- if (!implyOrder) {
- return aggregations;
- }
- ArrayList<ASTNode> wdwUDAFs = new ArrayList<ASTNode>();
- for(ASTNode function : aggregations) {
- String fnName = function.getChild(0).getText().toLowerCase();
- if ( FunctionRegistry.impliesOrder(fnName)) {
- wdwUDAFs.add(function);
- continue;
- }
- boolean hasLLInArgs = false;
- for(int i=1; i < function.getChildCount(); i++) {
- ASTNode child = (ASTNode) function.getChild(i);
- hasLLInArgs = containsLeadLagUDF(child);
- if (hasLLInArgs) {
- break;
- }
- }
- if (hasLLInArgs) {
- wdwUDAFs.add(function);
- }
- }
- return wdwUDAFs;
- }
-
- private void extractWindowingUDAFs(ASTNode expressionTree,
- ArrayList<ASTNode> aggregations) {
- int exprTokenType = expressionTree.getToken().getType();
- if (exprTokenType == HiveParser.TOK_FUNCTION
- || exprTokenType == HiveParser.TOK_FUNCTIONDI
- || exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
- assert (expressionTree.getChildCount() != 0);
- if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
- String functionName = unescapeIdentifier(expressionTree.getChild(0)
- .getText());
- WindowFunctionInfo fi = FunctionRegistry.getWindowFunctionInfo(functionName);
- if (fi != null) {
- aggregations.add(expressionTree);
- return;
- }
- }
- }
- for (int i = 0; i < expressionTree.getChildCount(); i++) {
- extractWindowingUDAFs((ASTNode) expressionTree.getChild(i),
- aggregations);
- }
- }
-
- private boolean containsLeadLagUDF(ASTNode expressionTree) {
- int exprTokenType = expressionTree.getToken().getType();
- if (exprTokenType == HiveParser.TOK_FUNCTION) {
- assert (expressionTree.getChildCount() != 0);
- if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
- String functionName = unescapeIdentifier(expressionTree.getChild(0)
- .getText());
- functionName = functionName.toLowerCase();
- if ( FunctionRegistry.LAG_FUNC_NAME.equals(functionName) ||
- FunctionRegistry.LEAD_FUNC_NAME.equals(functionName)
- ) {
- return true;
- }
- }
- }
- for (int i = 0; i < expressionTree.getChildCount(); i++) {
- if ( containsLeadLagUDF((ASTNode) expressionTree.getChild(i))) {
- return true;
- }
- }
- return false;
- }
-
- /*
- * - Invoked during Phase1 when a TOK_SELECT is encountered.
- * - Select tree form is: ^(TOK_SELECT ^(TOK_SELECTEXPR...) ^(TOK_SELECTEXPR...) ...)
- * - A Select Item form is: ^(TOK_SELEXPR selectExpression Identifier* window_specification?)
- *
- * See checkAndExtractWindowFunctionsInSelect for rules on what makes a UDAF invocation
- * a Windowing Function invocation
- */
- private void handleWindowingExprsInSelectList(QB qb, String dest, ASTNode selectNode)
- throws SemanticException {
- for(int i=0; i < selectNode.getChildCount(); i++)
- {
- ASTNode selectExpr = (ASTNode) selectNode.getChild(i);
- if ( selectExpr.getType() != HiveParser.TOK_SELEXPR )
- {
- continue;
- }
- boolean hasWindowingExprs = checkAndExtractWindowFunctionsInSelect(qb, selectExpr, dest);
-
- }
- }
-
private void handleQueryWindowClauses(QB qb, Phase1Ctx ctx_1, ASTNode node)
throws SemanticException {
WindowingSpec spec = qb.getWindowingSpec(ctx_1.dest);
@@ -9719,7 +9589,7 @@ public class SemanticAnalyzer extends Ba
ASTNode nameNode = (ASTNode) node.getChild(0);
wfSpec.setName(nameNode.getText());
- for(int i=1; i < node.getChildCount(); i++) {
+ for(int i=1; i < node.getChildCount()-1; i++) {
ASTNode child = (ASTNode) node.getChild(i);
wfSpec.addArg(child);
}
@@ -9729,18 +9599,32 @@ public class SemanticAnalyzer extends Ba
wfSpec.setWindowSpec(ws);
}
- /*
- * In order to distinguish between different UDAF invocations on the same UDAF but different Windows
- * add the WdwSpec node as a child of the Function Node.
- * It is safe to do this after the function node has been converetd to a WdwFuncSpec.
- */
- if ( wsNode != null ) {
- node.addChild(wsNode);
- }
-
return wfSpec;
}
+ private boolean containsLeadLagUDF(ASTNode expressionTree) {
+ int exprTokenType = expressionTree.getToken().getType();
+ if (exprTokenType == HiveParser.TOK_FUNCTION) {
+ assert (expressionTree.getChildCount() != 0);
+ if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
+ String functionName = unescapeIdentifier(expressionTree.getChild(0)
+ .getText());
+ functionName = functionName.toLowerCase();
+ if ( FunctionRegistry.LAG_FUNC_NAME.equals(functionName) ||
+ FunctionRegistry.LEAD_FUNC_NAME.equals(functionName)
+ ) {
+ return true;
+ }
+ }
+ }
+ for (int i = 0; i < expressionTree.getChildCount(); i++) {
+ if ( containsLeadLagUDF((ASTNode) expressionTree.getChild(i))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void processQueryWindowClause(WindowingSpec spec, ASTNode node)
throws SemanticException {
ASTNode nameNode = (ASTNode) node.getChild(0);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Mon Aug 26 21:42:12 2013
@@ -176,7 +176,8 @@ public final class TypeCheckProcFactory
private static Map<ASTNode, ExprNodeDesc> convert(Map<Node, Object> outputs) {
Map<ASTNode, ExprNodeDesc> converted = new LinkedHashMap<ASTNode, ExprNodeDesc>();
for (Map.Entry<Node, Object> entry : outputs.entrySet()) {
- if (entry.getKey() instanceof ASTNode && entry.getValue() instanceof ExprNodeDesc) {
+ if (entry.getKey() instanceof ASTNode &&
+ (entry.getValue() == null || entry.getValue() instanceof ExprNodeDesc)) {
converted.put((ASTNode)entry.getKey(), (ExprNodeDesc)entry.getValue());
} else {
LOG.warn("Invalid type entry " + entry);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java Mon Aug 26 21:42:12 2013
@@ -66,7 +66,7 @@ public class ExprNodeGenericFuncDesc ext
* This class uses a writableObjectInspector rather than a TypeInfo to store
* the canonical type information for this NodeDesc.
*/
- private ObjectInspector writableObjectInspector;
+ private transient ObjectInspector writableObjectInspector;
//Is this an expression that should perform a comparison for sorted searches
private boolean isSortedExpr;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java Mon Aug 26 21:42:12 2013
@@ -21,10 +21,11 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import java.util.TreeMap;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.parse.SplitSample;
/**
@@ -193,6 +194,36 @@ public class FetchWork implements Serial
}
/**
+ * Get Partition descriptors in sorted (ascending) order of partition directory
+ *
+ * @return the partDesc array list
+ */
+ @Explain(displayName = "Partition Description", normalExplain = false)
+ public ArrayList<PartitionDesc> getPartDescOrderedByPartDir() {
+ ArrayList<PartitionDesc> partDescOrdered = partDesc;
+
+ if (partDir != null && partDir.size() > 1) {
+ if (partDesc == null || partDir.size() != partDesc.size()) {
+ throw new RuntimeException(
+ "Partiton Directory list size doesn't match Partition Descriptor list size");
+ }
+
+ // Construct a sorted Map of Partition Dir - Partition Descriptor; ordering is based on
+ // patition dir (map key)
+ // Assumption: there is a 1-1 mapping between partition dir and partition descriptor lists
+ TreeMap<String, PartitionDesc> partDirToPartSpecMap = new TreeMap<String, PartitionDesc>();
+ for (int i = 0; i < partDir.size(); i++) {
+ partDirToPartSpecMap.put(partDir.get(i), partDesc.get(i));
+ }
+
+ // Extract partition desc from sorted map (ascending order of part dir)
+ partDescOrdered = new ArrayList<PartitionDesc>(partDirToPartSpecMap.values());
+ }
+
+ return partDescOrdered;
+ }
+
+ /**
* @return the partDescs for paths
*/
public List<PartitionDesc> getPartDescs(List<Path> paths) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java Mon Aug 26 21:42:12 2013
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
@@ -40,7 +39,7 @@ public class ListBucketingCtx implements
private static final long serialVersionUID = 1L;
private List<String> skewedColNames;
private List<List<String>> skewedColValues;
- private Map<SkewedValueList, String> lbLocationMap;
+ private Map<List<String>, String> lbLocationMap;
private List<SkewedColumnPositionPair> rowSkewedIndex;
private boolean isStoredAsSubDirectories;
private String defaultKey;
@@ -83,14 +82,14 @@ public class ListBucketingCtx implements
/**
* @return the lbLocationMap
*/
- public Map<SkewedValueList, String> getLbLocationMap() {
+ public Map<List<String>, String> getLbLocationMap() {
return lbLocationMap;
}
/**
* @param lbLocationMap the lbLocationMap to set
*/
- public void setLbLocationMap(Map<SkewedValueList, String> lbLocationMap) {
+ public void setLbLocationMap(Map<List<String>, String> lbLocationMap) {
this.lbLocationMap = lbLocationMap;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java Mon Aug 26 21:42:12 2013
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order;
@@ -51,8 +52,11 @@ public class PTFDesc extends AbstractOpe
*/
boolean isMapSide = false;
+ HiveConf cfg;
+
static{
PTFUtils.makeTransient(PTFDesc.class, "llInfo");
+ PTFUtils.makeTransient(PTFDesc.class, "cfg");
}
public PartitionedTableFunctionDef getFuncDef() {
@@ -87,6 +91,14 @@ public class PTFDesc extends AbstractOpe
this.isMapSide = isMapSide;
}
+ public HiveConf getCfg() {
+ return cfg;
+ }
+
+ public void setCfg(HiveConf cfg) {
+ this.cfg = cfg;
+ }
+
public abstract static class PTFInputDef {
String expressionTreeString;
ShapeDetails outputShape;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Mon Aug 26 21:42:12 2013
@@ -27,6 +27,7 @@ import java.util.Stack;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.PTFTranslator.LeadLagInfo;
import org.apache.hadoop.hive.ql.parse.WindowingExprNodeEvaluatorFactory;
@@ -71,6 +72,7 @@ public class PTFDeserializer {
public PTFDeserializer(PTFDesc ptfDesc, StructObjectInspector inputOI, HiveConf hConf) {
super();
this.ptfDesc = ptfDesc;
+ ptfDesc.setCfg(hConf);
this.inputOI = inputOI;
this.hConf = hConf;
llInfo = new LeadLagInfo();
@@ -292,7 +294,8 @@ public class PTFDeserializer {
SerDe serDe = (SerDe) SerDeUtils.lookupDeserializer(serdeClassName);
serDe.initialize(hConf, serDeProps);
shp.setSerde(serDe);
- shp.setOI((StructObjectInspector) serDe.getObjectInspector());
+ StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serDe, OI);
+ shp.setOI(outOI);
} catch (SerDeException se)
{
throw new HiveException(se);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java Mon Aug 26 21:42:12 2013
@@ -137,6 +137,10 @@ public class TableScanDesc extends Abstr
this.virtualCols.addAll(virtualCols);
}
+ public boolean hasVirtualCols() {
+ return virtualCols != null && !virtualCols.isEmpty();
+ }
+
public void setStatsAggPrefix(String k) {
statsAggKeyPrefix = k;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java Mon Aug 26 21:42:12 2013
@@ -57,10 +57,14 @@ public abstract class HiveAuthorizationP
this.handler = handler;
}
+ public boolean isRunFromMetaStore(){
+ return (this.hiveClient == null);
+ }
+
public PrincipalPrivilegeSet get_privilege_set(HiveObjectType column, String dbName,
String tableName, List<String> partValues, String col, String userName,
List<String> groupNames) throws HiveException {
- if (hiveClient != null) {
+ if (!isRunFromMetaStore()) {
return hiveClient.get_privilege_set(
column, dbName, tableName, partValues, col, userName, groupNames);
} else {
@@ -77,7 +81,7 @@ public abstract class HiveAuthorizationP
}
public Database getDatabase(String dbName) throws HiveException {
- if (hiveClient != null) {
+ if (!isRunFromMetaStore()) {
return hiveClient.getDatabase(dbName);
} else {
try {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Mon Aug 26 21:42:12 2013
@@ -63,6 +63,22 @@ public class StorageBasedAuthorizationPr
private Warehouse wh;
+ /**
+ * Make sure that the warehouse variable is set up properly.
+ * @throws MetaException if unable to instantiate
+ */
+ private void initWh() throws MetaException {
+ if (wh == null){
+ if(!hive_db.isRunFromMetaStore()){
+ this.wh = new Warehouse(getConf());
+ }else{
+ // not good if we reach here, this was initialized at setMetaStoreHandler() time.
+ // this means handler.getWh() is returning null. Error out.
+ throw new IllegalStateException("Unitialized Warehouse from MetastoreHandler");
+ }
+ }
+ }
+
@Override
public void init(Configuration conf) throws HiveException {
hive_db = new HiveProxy();
@@ -100,6 +116,7 @@ public class StorageBasedAuthorizationPr
// we try to determine what the path would be after the create table is issued.
Path path = null;
try {
+ initWh();
String location = table.getTTable().getSd().getLocation();
if (location == null || location.isEmpty()) {
path = wh.getTablePath(hive_db.getDatabase(table.getDbName()), table.getTableName());
@@ -305,6 +322,7 @@ public class StorageBasedAuthorizationPr
protected Path getDbLocation(Database db) throws HiveException {
try {
+ initWh();
String location = db.getLocationUri();
if (location == null) {
return wh.getDefaultDatabasePath(db.getName());
@@ -317,15 +335,11 @@ public class StorageBasedAuthorizationPr
}
private HiveException hiveException(Exception e) {
- HiveException ex = new HiveException(e);
- ex.initCause(e);
- return ex;
+ return new HiveException(e);
}
private AuthorizationException authorizationException(Exception e) {
- AuthorizationException ex = new AuthorizationException(e);
- ex.initCause(e);
- return ex;
+ return new AuthorizationException(e);
}
private static AccessControlException accessControlException(
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Mon Aug 26 21:42:12 2013
@@ -45,6 +45,8 @@ import org.apache.hadoop.hive.ql.MapRedS
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.history.HiveHistory;
+import org.apache.hadoop.hive.ql.history.HiveHistoryImpl;
+import org.apache.hadoop.hive.ql.history.HiveHistoryProxyHandler;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
@@ -255,19 +257,19 @@ public class SessionState {
tss.set(startSs);
- if (startSs.hiveHist == null) {
- startSs.hiveHist = new HiveHistory(startSs);
+ if(startSs.hiveHist == null){
+ if (startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) {
+ startSs.hiveHist = new HiveHistoryImpl(startSs);
+ }else {
+ //Hive history is disabled, create a no-op proxy
+ startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy();
+ }
}
if (startSs.getTmpOutputFile() == null) {
- // per-session temp file containing results to be sent from HiveServer to HiveClient
- File tmpDir = new File(
- HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVEHISTORYFILELOC));
- String sessionID = startSs.getConf().getVar(HiveConf.ConfVars.HIVESESSIONID);
+ // set temp file containing results to be sent to HiveClient
try {
- File tmpFile = File.createTempFile(sessionID, ".pipeout", tmpDir);
- tmpFile.deleteOnExit();
- startSs.setTmpOutputFile(tmpFile);
+ startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -299,6 +301,33 @@ public class SessionState {
}
/**
+ * @param conf
+ * @return per-session temp file
+ * @throws IOException
+ */
+ private static File createTempFile(HiveConf conf) throws IOException {
+ String lScratchDir =
+ HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR);
+
+ File tmpDir = new File(lScratchDir);
+ String sessionID = conf.getVar(HiveConf.ConfVars.HIVESESSIONID);
+ if (!tmpDir.exists()) {
+ if (!tmpDir.mkdirs()) {
+ //Do another exists to check to handle possible race condition
+ // Another thread might have created the dir, if that is why
+ // mkdirs returned false, that is fine
+ if(!tmpDir.exists()){
+ throw new RuntimeException("Unable to create log directory "
+ + lScratchDir);
+ }
+ }
+ }
+ File tmpFile = File.createTempFile(sessionID, ".pipeout", tmpDir);
+ tmpFile.deleteOnExit();
+ return tmpFile;
+ }
+
+ /**
* get the current session.
*/
public static SessionState get() {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java Mon Aug 26 21:42:12 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -29,10 +30,10 @@ import org.apache.hadoop.io.Text;
*
*/
@Description(name = "hex",
- value = "_FUNC_(n or str) - Convert the argument to hexadecimal ",
+ value = "_FUNC_(n, bin, or str) - Convert the argument to hexadecimal ",
extended = "If the argument is a string, returns two hex digits for each "
+ "character in the string.\n"
- + "If the argument is a number, returns the hexadecimal representation.\n"
+ + "If the argument is a number or binary, returns the hexadecimal representation.\n"
+ "Example:\n"
+ " > SELECT _FUNC_(17) FROM src LIMIT 1;\n"
+ " 'H1'\n"
@@ -44,7 +45,7 @@ public class UDFHex extends UDF {
/**
* Convert num to hex.
- *
+ *
*/
private Text evaluate(long num) {
// Extract the hex digits of num into value[] from right to left
@@ -76,26 +77,41 @@ public class UDFHex extends UDF {
/**
* Convert every character in s to two hex digits.
- *
+ *
*/
public Text evaluate(Text s) {
if (s == null) {
return null;
}
- if (value.length < s.getLength() * 2) {
- value = new byte[s.getLength() * 2];
+ byte[] str = s.getBytes();
+ return evaluate(str, s.getLength());
+ }
+
+ /**
+ * Convert bytes to a hex string
+ */
+ public Text evaluate(BytesWritable b){
+ if (b == null) {
+ return null;
}
- byte[] str = s.getBytes();
- for (int i = 0; i < s.getLength(); i++) {
+ byte[] bytes = b.getBytes();
+ return evaluate(bytes, b.getLength());
+ }
+
+ private Text evaluate(byte[] bytes, int length){
+ if (value.length < length * 2) {
+ value = new byte[length * 2];
+ }
+
+ for (int i = 0; i < length; i++) {
value[i * 2] = (byte) Character.toUpperCase(Character.forDigit(
- (str[i] & 0xF0) >>> 4, 16));
+ (bytes[i] & 0xF0) >>> 4, 16));
value[i * 2 + 1] = (byte) Character.toUpperCase(Character.forDigit(
- str[i] & 0x0F, 16));
+ bytes[i] & 0x0F, 16));
}
-
- result.set(value, 0, s.getLength() * 2);
+ result.set(value, 0, length*2);
return result;
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java Mon Aug 26 21:42:12 2013
@@ -27,18 +27,14 @@ import org.apache.hadoop.io.Text;
*
*/
@Description(name = "unhex",
- value = "_FUNC_(str) - Converts hexadecimal argument to string",
+ value = "_FUNC_(str) - Converts hexadecimal argument to binary",
extended = "Performs the inverse operation of HEX(str). That is, it interprets\n"
+ "each pair of hexadecimal digits in the argument as a number and\n"
- + "converts it to the character represented by the number. The\n"
+ + "converts it to the byte representation of the number. The\n"
+ "resulting characters are returned as a binary string.\n\n"
+ "Example:\n"
- + "> SELECT UNHEX('4D7953514C') from src limit 1;\n"
- + "'MySQL'\n"
- + "> SELECT UNHEX(HEX('string')) from src limit 1;\n"
- + "'string'\n"
- + "> SELECT HEX(UNHEX('1267')) from src limit 1;\n"
- + "'1267'\n\n"
+ + "> SELECT DECODE(UNHEX('4D7953514C'), 'UTF-8') from src limit 1;\n"
+ + "'MySQL'\n\n"
+ "The characters in the argument string must be legal hexadecimal\n"
+ "digits: '0' .. '9', 'A' .. 'F', 'a' .. 'f'. If UNHEX() encounters\n"
+ "any nonhexadecimal digits in the argument, it returns NULL. Also,\n"
@@ -47,9 +43,9 @@ public class UDFUnhex extends UDF {
/**
* Convert every two hex digits in s into.
- *
+ *
*/
- public Text evaluate(Text s) {
+ public byte[] evaluate(Text s) {
if (s == null) {
return null;
}
@@ -72,6 +68,6 @@ public class UDFUnhex extends UDF {
}
}
- return new Text(result);
+ return result;
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java Mon Aug 26 21:42:12 2013
@@ -30,14 +30,13 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput;
import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector;
import org.apache.hadoop.io.LongWritable;
/**
@@ -46,8 +45,7 @@ import org.apache.hadoop.io.LongWritable
*/
abstract public class AbstractGenericUDFEWAHBitmapBop extends GenericUDF {
protected final ArrayList<Object> ret = new ArrayList<Object>();
- private ObjectInspector b1OI;
- private ObjectInspector b2OI;
+ private transient ObjectInspector b1OI;
private final String name;
AbstractGenericUDFEWAHBitmapBop(String name) {
@@ -70,15 +68,13 @@ abstract public class AbstractGenericUDF
+ arguments[0].getTypeName() + "\" is found");
}
- if (arguments[1].getCategory().equals(Category.LIST)) {
- b2OI = (ListObjectInspector) arguments[1];
- } else {
+ if (!arguments[1].getCategory().equals(Category.LIST)) {
throw new UDFArgumentTypeException(1, "\""
+ Category.LIST.toString().toLowerCase()
+ "\" is expected at function " + name + ", but \""
+ arguments[1].getTypeName() + "\" is found");
- }
+ }
return ObjectInspectorFactory
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory
.writableLongObjectInspector);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java Mon Aug 26 21:42:12 2013
@@ -26,18 +26,18 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.*;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
/**
* common class for reflective UDFs
*/
public abstract class AbstractGenericUDFReflect extends GenericUDF {
- private PrimitiveObjectInspector[] parameterOIs;
- private PrimitiveTypeEntry[] parameterTypes;
- private Class[] parameterClasses;
+ private transient PrimitiveObjectInspector[] parameterOIs;
+ private transient PrimitiveTypeEntry[] parameterTypes;
+ private transient Class[] parameterClasses;
- private Object[] parameterJavaValues;
+ private transient Object[] parameterJavaValues;
void setupParameterOIs(ObjectInspector[] arguments, int start) throws UDFArgumentTypeException {
int length = arguments.length - start;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java Mon Aug 26 21:42:12 2013
@@ -243,15 +243,15 @@ public class GenericUDAFAverage extends
public static abstract class AbstractGenericUDAFAverageEvaluator<TYPE> extends GenericUDAFEvaluator {
// For PARTIAL1 and COMPLETE
- private PrimitiveObjectInspector inputOI;
+ private transient PrimitiveObjectInspector inputOI;
// For PARTIAL2 and FINAL
- private StructObjectInspector soi;
- private StructField countField;
- private StructField sumField;
+ private transient StructObjectInspector soi;
+ private transient StructField countField;
+ private transient StructField sumField;
private LongObjectInspector countFieldOI;
private ObjectInspector sumFieldOI;
// For PARTIAL1 and PARTIAL2
- protected Object[] partialResult;
+ protected transient Object[] partialResult;
private boolean warned = false;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java Mon Aug 26 21:42:12 2013
@@ -69,10 +69,11 @@ public class GenericUDAFCollectSet exten
private PrimitiveObjectInspector inputOI;
// For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations (list
// of objs)
- private StandardListObjectInspector loi;
+ private transient StandardListObjectInspector loi;
- private StandardListObjectInspector internalMergeOI;
+ private transient StandardListObjectInspector internalMergeOI;
+ @Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters)
throws HiveException {
super.init(m, parameters);