You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/10/06 05:44:26 UTC
svn commit: r1629562 [10/38] - in /hive/branches/spark: ./ accumulo-handler/
beeline/ beeline/src/java/org/apache/hive/beeline/ bin/ext/ common/
common/src/java/org/apache/hadoop/hive/conf/
common/src/test/org/apache/hadoop/hive/common/type/ contrib/sr...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Mon Oct 6 03:44:13 2014
@@ -26,8 +26,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -280,11 +278,6 @@ public abstract class TaskCompiler {
tsk.setRetryCmdWhenFail(true);
}
}
-
- Interner<TableDesc> interner = Interners.newStrongInterner();
- for (Task<? extends Serializable> rootTask : rootTasks) {
- GenMapRedUtils.internTableDesc(rootTask, interner);
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Mon Oct 6 03:44:13 2014
@@ -36,9 +36,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
-import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
-import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -64,7 +62,6 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate;
import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin;
import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
-import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc;
import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
@@ -333,17 +330,10 @@ public class TezCompiler extends TaskCom
opRules.put(new RuleRegExp("No more walking on ReduceSink-MapJoin",
MapJoinOperator.getOperatorName() + "%"), new ReduceSinkMapJoinProc());
- opRules.put(new RuleRegExp("Recoginze a Sorted Merge Join operator to setup the right edge and"
- + " stop traversing the DummyStore-MapJoin", CommonMergeJoinOperator.getOperatorName()
- + "%"), new MergeJoinProc());
-
opRules.put(new RuleRegExp("Split Work + Move/Merge - FileSink",
FileSinkOperator.getOperatorName() + "%"),
new CompositeProcessor(new FileSinkProcessor(), genTezWork));
- opRules.put(new RuleRegExp("Split work - DummyStore", DummyStoreOperator.getOperatorName()
- + "%"), genTezWork);
-
opRules.put(new RuleRegExp("Handle Potential Analyze Command",
TableScanOperator.getOperatorName() + "%"),
new ProcessAnalyzeTable(GenTezUtils.getUtils()));
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Mon Oct 6 03:44:13 2014
@@ -54,19 +54,6 @@ public class TypeCheckCtx implements Nod
private boolean allowDistinctFunctions;
- private final boolean allowGBExprElimination;
-
- private final boolean allowAllColRef;
-
- private final boolean allowFunctionStar;
-
- private final boolean allowWindowing;
-
- // "[]" : LSQUARE/INDEX Expression
- private final boolean allowIndexExpr;
-
- private final boolean allowSubQueryExpr;
-
/**
* Constructor.
*
@@ -74,23 +61,10 @@ public class TypeCheckCtx implements Nod
* The input row resolver of the previous operator.
*/
public TypeCheckCtx(RowResolver inputRR) {
- this(inputRR, false, true, true, true, true, true, true, true);
- }
-
- public TypeCheckCtx(RowResolver inputRR, boolean allowStatefulFunctions,
- boolean allowDistinctFunctions, boolean allowGBExprElimination, boolean allowAllColRef,
- boolean allowFunctionStar, boolean allowWindowing,
- boolean allowIndexExpr, boolean allowSubQueryExpr) {
setInputRR(inputRR);
error = null;
- this.allowStatefulFunctions = allowStatefulFunctions;
- this.allowDistinctFunctions = allowDistinctFunctions;
- this.allowGBExprElimination = allowGBExprElimination;
- this.allowAllColRef = allowAllColRef;
- this.allowFunctionStar = allowFunctionStar;
- this.allowWindowing = allowWindowing;
- this.allowIndexExpr = allowIndexExpr;
- this.allowSubQueryExpr = allowSubQueryExpr;
+ allowStatefulFunctions = false;
+ allowDistinctFunctions = true;
}
/**
@@ -124,8 +98,7 @@ public class TypeCheckCtx implements Nod
}
/**
- * @param allowStatefulFunctions
- * whether to allow stateful UDF invocations
+ * @param allowStatefulFunctions whether to allow stateful UDF invocations
*/
public void setAllowStatefulFunctions(boolean allowStatefulFunctions) {
this.allowStatefulFunctions = allowStatefulFunctions;
@@ -163,31 +136,7 @@ public class TypeCheckCtx implements Nod
this.allowDistinctFunctions = allowDistinctFunctions;
}
- public boolean getAllowDistinctFunctions() {
+ public boolean isAllowDistinctFunctions() {
return allowDistinctFunctions;
}
-
- public boolean getAllowGBExprElimination() {
- return allowGBExprElimination;
- }
-
- public boolean getallowAllColRef() {
- return allowAllColRef;
- }
-
- public boolean getallowFunctionStar() {
- return allowFunctionStar;
- }
-
- public boolean getallowWindowing() {
- return allowWindowing;
- }
-
- public boolean getallowIndexExpr() {
- return allowIndexExpr;
- }
-
- public boolean getallowSubQueryExpr() {
- return allowSubQueryExpr;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Mon Oct 6 03:44:13 2014
@@ -80,12 +80,12 @@ import org.apache.hadoop.hive.serde2.typ
* expression Node Descriptor trees. They also introduce the correct conversion
* functions to do proper implicit conversion.
*/
-public class TypeCheckProcFactory {
+public final class TypeCheckProcFactory {
protected static final Log LOG = LogFactory.getLog(TypeCheckProcFactory.class
.getName());
- protected TypeCheckProcFactory() {
+ private TypeCheckProcFactory() {
// prevent instantiation
}
@@ -118,7 +118,7 @@ public class TypeCheckProcFactory {
RowResolver input = ctx.getInputRR();
ExprNodeDesc desc = null;
- if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) {
+ if ((ctx == null) || (input == null)) {
return null;
}
@@ -137,13 +137,8 @@ public class TypeCheckProcFactory {
return desc;
}
- public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr, TypeCheckCtx tcCtx)
- throws SemanticException {
- return genExprNode(expr, tcCtx, new TypeCheckProcFactory());
- }
-
- protected static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr,
- TypeCheckCtx tcCtx, TypeCheckProcFactory tf) throws SemanticException {
+ public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr,
+ TypeCheckCtx tcCtx) throws SemanticException {
// Create the walker, the rules dispatcher and the context.
// create a walker which walks the tree in a DFS manner while maintaining
// the operator stack. The dispatcher
@@ -151,13 +146,13 @@ public class TypeCheckProcFactory {
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"),
- tf.getNullExprProcessor());
+ getNullExprProcessor());
opRules.put(new RuleRegExp("R2", HiveParser.Number + "%|" +
HiveParser.TinyintLiteral + "%|" +
HiveParser.SmallintLiteral + "%|" +
HiveParser.BigintLiteral + "%|" +
HiveParser.DecimalLiteral + "%"),
- tf.getNumExprProcessor());
+ getNumExprProcessor());
opRules
.put(new RuleRegExp("R3", HiveParser.Identifier + "%|"
+ HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL + "%|"
@@ -167,18 +162,18 @@ public class TypeCheckProcFactory {
+ HiveParser.KW_ARRAY + "%|" + HiveParser.KW_MAP + "%|"
+ HiveParser.KW_STRUCT + "%|" + HiveParser.KW_EXISTS + "%|"
+ HiveParser.TOK_SUBQUERY_OP_NOTIN + "%"),
- tf.getStrExprProcessor());
+ getStrExprProcessor());
opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
- + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor());
- opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), tf.getDateExprProcessor());
+ + HiveParser.KW_FALSE + "%"), getBoolExprProcessor());
+ opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), getDateExprProcessor());
opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"),
- tf.getColumnExprProcessor());
+ getColumnExprProcessor());
opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"),
- tf.getSubQueryExprProcessor());
+ getSubQueryExprProcessor());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
- Dispatcher disp = new DefaultRuleDispatcher(tf.getDefaultExprProcessor(),
+ Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(),
opRules, tcCtx);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -234,7 +229,7 @@ public class TypeCheckProcFactory {
*
* @return NullExprProcessor.
*/
- public NullExprProcessor getNullExprProcessor() {
+ public static NullExprProcessor getNullExprProcessor() {
return new NullExprProcessor();
}
@@ -309,7 +304,7 @@ public class TypeCheckProcFactory {
*
* @return NumExprProcessor.
*/
- public NumExprProcessor getNumExprProcessor() {
+ public static NumExprProcessor getNumExprProcessor() {
return new NumExprProcessor();
}
@@ -367,7 +362,7 @@ public class TypeCheckProcFactory {
*
* @return StrExprProcessor.
*/
- public StrExprProcessor getStrExprProcessor() {
+ public static StrExprProcessor getStrExprProcessor() {
return new StrExprProcessor();
}
@@ -413,7 +408,7 @@ public class TypeCheckProcFactory {
*
* @return BoolExprProcessor.
*/
- public BoolExprProcessor getBoolExprProcessor() {
+ public static BoolExprProcessor getBoolExprProcessor() {
return new BoolExprProcessor();
}
@@ -454,7 +449,7 @@ public class TypeCheckProcFactory {
*
* @return DateExprProcessor.
*/
- public DateExprProcessor getDateExprProcessor() {
+ public static DateExprProcessor getDateExprProcessor() {
return new DateExprProcessor();
}
@@ -551,7 +546,7 @@ public class TypeCheckProcFactory {
*
* @return ColumnExprProcessor.
*/
- public ColumnExprProcessor getColumnExprProcessor() {
+ public static ColumnExprProcessor getColumnExprProcessor() {
return new ColumnExprProcessor();
}
@@ -618,7 +613,7 @@ public class TypeCheckProcFactory {
windowingTokens.add(HiveParser.TOK_TABSORTCOLNAMEDESC);
}
- protected static boolean isRedundantConversionFunction(ASTNode expr,
+ private static boolean isRedundantConversionFunction(ASTNode expr,
boolean isFunction, ArrayList<ExprNodeDesc> children) {
if (!isFunction) {
return false;
@@ -705,30 +700,7 @@ public class TypeCheckProcFactory {
return getFuncExprNodeDescWithUdfData(udfName, null, children);
}
- protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, FunctionInfo fi,
- List<ExprNodeDesc> children, GenericUDF genericUDF) throws SemanticException {
- // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't
- // supported
- if (fi.getGenericUDTF() != null) {
- throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg());
- }
- // UDAF in filter condition, group-by caluse, param of funtion, etc.
- if (fi.getGenericUDAFResolver() != null) {
- if (isFunction) {
- throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg((ASTNode) expr
- .getChild(0)));
- } else {
- throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr));
- }
- }
- if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) {
- if (FunctionRegistry.isStateful(genericUDF)) {
- throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg());
- }
- }
- }
-
- protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
+ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
boolean isFunction, ArrayList<ExprNodeDesc> children, TypeCheckCtx ctx)
throws SemanticException, UDFArgumentException {
// return the child directly if the conversion is redundant.
@@ -741,7 +713,6 @@ public class TypeCheckProcFactory {
ExprNodeDesc desc;
if (funcText.equals(".")) {
// "." : FIELD Expression
-
assert (children.size() == 2);
// Only allow constant field name for now
assert (children.get(1) instanceof ExprNodeConstantDesc);
@@ -756,22 +727,23 @@ public class TypeCheckProcFactory {
// Allow accessing a field of list element structs directly from a list
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
if (isList) {
- objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo();
+ objectTypeInfo = ((ListTypeInfo) objectTypeInfo)
+ .getListElementTypeInfo();
}
if (objectTypeInfo.getCategory() != Category.STRUCT) {
throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
}
- TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString);
+ TypeInfo t = ((StructTypeInfo) objectTypeInfo)
+ .getStructFieldTypeInfo(fieldNameString);
if (isList) {
t = TypeInfoFactory.getListTypeInfo(t);
}
- desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
+ desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString,
+ isList);
+
} else if (funcText.equals("[")) {
// "[]" : LSQUARE/INDEX Expression
- if (!ctx.getallowIndexExpr())
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr));
-
assert (children.size() == 2);
// Check whether this is a list or a map
@@ -787,7 +759,8 @@ public class TypeCheckProcFactory {
// Calculate TypeInfo
TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo();
- desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children);
+ desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry
+ .getGenericUDFForIndex(), children);
} else if (myt.getCategory() == Category.MAP) {
if (!FunctionRegistry.implicitConvertible(children.get(1).getTypeInfo(),
((MapTypeInfo) myt).getMapKeyTypeInfo())) {
@@ -796,9 +769,11 @@ public class TypeCheckProcFactory {
}
// Calculate TypeInfo
TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo();
- desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children);
+ desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry
+ .getGenericUDFForIndex(), children);
} else {
- throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, myt.getTypeName()));
+ throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr,
+ myt.getTypeName()));
}
} else {
// other operators or functions
@@ -850,7 +825,26 @@ public class TypeCheckProcFactory {
}
}
- validateUDF(expr, isFunction, ctx, fi, children, genericUDF);
+ // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't
+ // supported
+ if (fi.getGenericUDTF() != null) {
+ throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg());
+ }
+ // UDAF in filter condition, group-by caluse, param of funtion, etc.
+ if (fi.getGenericUDAFResolver() != null) {
+ if (isFunction) {
+ throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.
+ getMsg((ASTNode) expr.getChild(0)));
+ } else {
+ throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr));
+ }
+ }
+ if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) {
+ if (FunctionRegistry.isStateful(genericUDF)) {
+ throw new SemanticException(
+ ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg());
+ }
+ }
// Try to infer the type of the constant only if there are two
// nodes, one of them is column and the other is numeric const
@@ -961,24 +955,6 @@ public class TypeCheckProcFactory {
return false;
}
- protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr,
- Object... nodeOutputs) throws SemanticException {
- RowResolver input = ctx.getInputRR();
- String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0)
- .getText());
- // NOTE: tableAlias must be a valid non-ambiguous table alias,
- // because we've checked that in TOK_TABLE_OR_COL's process method.
- ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue()
- .toString());
-
- if (colInfo == null) {
- ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
- return null;
- }
- return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(),
- colInfo.getTabAlias(), colInfo.getIsVirtualCol());
- }
-
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
@@ -1028,11 +1004,7 @@ public class TypeCheckProcFactory {
* The difference is that there is translation for Window related tokens, so we just
* return null;
*/
- if (windowingTokens.contains(expr.getType())) {
- if (!ctx.getallowWindowing())
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
- ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context")));
-
+ if ( windowingTokens.contains(expr.getType())) {
return null;
}
@@ -1041,11 +1013,6 @@ public class TypeCheckProcFactory {
}
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
- if (!ctx.getallowAllColRef())
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
- ErrorMsg.INVALID_COLUMN
- .getMsg("All column reference is not supported in the context")));
-
RowResolver input = ctx.getInputRR();
ExprNodeColumnListDesc columnList = new ExprNodeColumnListDesc();
assert expr.getChildCount() <= 1;
@@ -1083,7 +1050,22 @@ public class TypeCheckProcFactory {
if (expr.getType() == HiveParser.DOT
&& expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
&& nodeOutputs[0] == null) {
- return processQualifiedColRef(ctx, expr, nodeOutputs);
+
+ RowResolver input = ctx.getInputRR();
+ String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr
+ .getChild(0).getChild(0).getText());
+ // NOTE: tableAlias must be a valid non-ambiguous table alias,
+ // because we've checked that in TOK_TABLE_OR_COL's process method.
+ ColumnInfo colInfo = input.get(tableAlias,
+ ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString());
+
+ if (colInfo == null) {
+ ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
+ return null;
+ }
+ return new ExprNodeColumnDesc(colInfo.getType(), colInfo
+ .getInternalName(), colInfo.getTabAlias(), colInfo
+ .getIsVirtualCol());
}
// Return nulls for conversion operators
@@ -1098,7 +1080,7 @@ public class TypeCheckProcFactory {
expr.getType() == HiveParser.TOK_FUNCTIONSTAR ||
expr.getType() == HiveParser.TOK_FUNCTIONDI);
- if (!ctx.getAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) {
+ if (!ctx.isAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) {
throw new SemanticException(
SemanticAnalyzer.generateErrorMessage(expr, ErrorMsg.DISTINCT_NOT_SUPPORTED.getMsg()));
}
@@ -1117,11 +1099,6 @@ public class TypeCheckProcFactory {
}
if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) {
- if (!ctx.getallowFunctionStar())
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
- ErrorMsg.INVALID_COLUMN
- .getMsg(".* reference is not supported in the context")));
-
RowResolver input = ctx.getInputRR();
for (ColumnInfo colInfo : input.getColumnInfos()) {
if (!colInfo.getIsVirtualCol()) {
@@ -1134,7 +1111,8 @@ public class TypeCheckProcFactory {
// If any of the children contains null, then return a null
// this is a hack for now to handle the group by case
if (children.contains(null)) {
- List<String> possibleColumnNames = getReferenceableColumnAliases(ctx);
+ RowResolver input = ctx.getInputRR();
+ List<String> possibleColumnNames = input.getReferenceableColumnAliases(null, -1);
String reason = String.format("(possible column names are: %s)",
StringUtils.join(possibleColumnNames, ", "));
ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(0), reason),
@@ -1157,9 +1135,6 @@ public class TypeCheckProcFactory {
}
}
- protected List<String> getReferenceableColumnAliases(TypeCheckCtx ctx) {
- return ctx.getInputRR().getReferenceableColumnAliases(null, -1);
- }
}
/**
@@ -1167,7 +1142,7 @@ public class TypeCheckProcFactory {
*
* @return DefaultExprProcessor.
*/
- public DefaultExprProcessor getDefaultExprProcessor() {
+ public static DefaultExprProcessor getDefaultExprProcessor() {
return new DefaultExprProcessor();
}
@@ -1185,18 +1160,13 @@ public class TypeCheckProcFactory {
return null;
}
- ASTNode expr = (ASTNode) nd;
- ASTNode sqNode = (ASTNode) expr.getParent().getChild(1);
-
- if (!ctx.getallowSubQueryExpr())
- throw new SemanticException(SemanticAnalyzer.generateErrorMessage(sqNode,
- ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg()));
-
ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
if (desc != null) {
return desc;
}
+ ASTNode expr = (ASTNode) nd;
+ ASTNode sqNode = (ASTNode) expr.getParent().getChild(1);
/*
* Restriction.1.h :: SubQueries only supported in the SQL Where Clause.
*/
@@ -1212,7 +1182,7 @@ public class TypeCheckProcFactory {
*
* @return DateExprProcessor.
*/
- public SubQueryExprProcessor getSubQueryExprProcessor() {
+ public static SubQueryExprProcessor getSubQueryExprProcessor() {
return new SubQueryExprProcessor();
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java Mon Oct 6 03:44:13 2014
@@ -343,10 +343,8 @@ public class UpdateDeleteSemanticAnalyze
// Add the setRCols to the input list
for (String colName : setRCols) {
- if(columnAccessInfo != null) {//assuming this means we are not doing Auth
- columnAccessInfo.add(Table.getCompleteName(mTable.getDbName(), mTable.getTableName()),
+ columnAccessInfo.add(Table.getCompleteName(mTable.getDbName(), mTable.getTableName()),
colName);
- }
}
}
@@ -388,7 +386,7 @@ public class UpdateDeleteSemanticAnalyze
setRCols.add(colName.getText());
} else if (node.getChildren() != null) {
for (Node n : node.getChildren()) {
- addSetRCols((ASTNode)n, setRCols);
+ addSetRCols(node, setRCols);
}
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java Mon Oct 6 03:44:13 2014
@@ -110,12 +110,10 @@ public class AlterTableDesc extends DDLD
* @param newComment
* @param newType
*/
- public AlterTableDesc(String tblName, HashMap<String, String> partSpec,
- String oldColName, String newColName,
+ public AlterTableDesc(String tblName, String oldColName, String newColName,
String newType, String newComment, boolean first, String afterCol) {
super();
oldName = tblName;
- this.partSpec = partSpec;
this.oldColName = oldColName;
this.newColName = newColName;
newColType = newType;
@@ -144,12 +142,11 @@ public class AlterTableDesc extends DDLD
* @param newCols
* new columns to be added
*/
- public AlterTableDesc(String name, HashMap<String, String> partSpec, List<FieldSchema> newCols,
+ public AlterTableDesc(String name, List<FieldSchema> newCols,
AlterTableTypes alterType) {
op = alterType;
oldName = name;
this.newCols = new ArrayList<FieldSchema>(newCols);
- this.partSpec = partSpec;
}
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Mon Oct 6 03:44:13 2014
@@ -41,7 +41,6 @@ public abstract class BaseWork extends A
// Their function is mainly as root ops to give the mapjoin the correct
// schema info.
List<HashTableDummyOperator> dummyOps;
- int tag;
public BaseWork() {}
@@ -101,7 +100,7 @@ public abstract class BaseWork extends A
// add all children
opStack.addAll(opSet);
-
+
while(!opStack.empty()) {
Operator<?> op = opStack.pop();
returnSet.add(op);
@@ -140,12 +139,4 @@ public abstract class BaseWork extends A
}
public abstract void configureJobConf(JobConf job);
-
- public void setTag(int tag) {
- this.tag = tag;
- }
-
- public int getTag() {
- return tag;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Mon Oct 6 03:44:13 2014
@@ -52,7 +52,6 @@ public class DropTableDesc extends DDLDe
ArrayList<PartSpec> partSpecs;
boolean expectView;
boolean ifExists;
- boolean ifPurge;
boolean ignoreProtection;
public DropTableDesc() {
@@ -60,14 +59,12 @@ public class DropTableDesc extends DDLDe
/**
* @param tableName
- * @param ifPurge
*/
- public DropTableDesc(String tableName, boolean expectView, boolean ifExists, boolean ifPurge) {
+ public DropTableDesc(String tableName, boolean expectView, boolean ifExists) {
this.tableName = tableName;
this.partSpecs = null;
this.expectView = expectView;
this.ifExists = ifExists;
- this.ifPurge = ifPurge;
this.ignoreProtection = false;
}
@@ -152,19 +149,4 @@ public class DropTableDesc extends DDLDe
public void setIfExists(boolean ifExists) {
this.ifExists = ifExists;
}
-
- /**
- * @return whether Purge was specified
- */
- public boolean getIfPurge() {
- return ifPurge;
- }
-
- /**
- * @param ifPurge
- * set whether Purge was specified
- */
- public void setIfPurge(boolean ifPurge) {
- this.ifPurge = ifPurge;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Mon Oct 6 03:44:13 2014
@@ -25,7 +25,6 @@ import org.apache.hadoop.hive.serde.serd
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -94,7 +93,7 @@ public class ExprNodeConstantDesc extend
return "null";
}
- if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) || typeInfo instanceof BaseCharTypeInfo) {
+ if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
return "'" + value.toString() + "'";
} else if (typeInfo.getTypeName().equals(serdeConstants.BINARY_TYPE_NAME)) {
byte[] bytes = (byte[]) value;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Mon Oct 6 03:44:13 2014
@@ -372,42 +372,5 @@ public class ExprNodeDescUtils {
} catch (Exception e) {
return null;
}
- }
-
- public static void getExprNodeColumnDesc(List<ExprNodeDesc> exprDescList,
- Map<Integer, ExprNodeDesc> hashCodeTocolumnDescMap) {
- for (ExprNodeDesc exprNodeDesc : exprDescList) {
- getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap);
- }
- }
-
- /**
- * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc.
- *
- * @param exprDesc
- * @param hashCodeTocolumnDescMap
- * Assumption: If two ExprNodeColumnDesc have same hash code then
- * they are logically referring to same projection
- */
- public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc,
- Map<Integer, ExprNodeDesc> hashCodeTocolumnDescMap) {
- if (exprDesc instanceof ExprNodeColumnDesc) {
- hashCodeTocolumnDescMap.put(
- ((ExprNodeColumnDesc) exprDesc).hashCode(),
- ((ExprNodeColumnDesc) exprDesc));
- } else if (exprDesc instanceof ExprNodeColumnListDesc) {
- for (ExprNodeDesc child : ((ExprNodeColumnListDesc) exprDesc)
- .getChildren()) {
- getExprNodeColumnDesc(child, hashCodeTocolumnDescMap);
- }
- } else if (exprDesc instanceof ExprNodeGenericFuncDesc) {
- for (ExprNodeDesc child : ((ExprNodeGenericFuncDesc) exprDesc)
- .getChildren()) {
- getExprNodeColumnDesc(child, hashCodeTocolumnDescMap);
- }
- } else if (exprDesc instanceof ExprNodeFieldDesc) {
- getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(),
- hashCodeTocolumnDescMap);
- }
- }
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java Mon Oct 6 03:44:13 2014
@@ -69,7 +69,6 @@ public class MapJoinDesc extends JoinDes
// Hash table memory usage allowed; used in case of non-staged mapjoin.
private float hashtableMemoryUsage;
- protected boolean genJoinKeys = true;
public MapJoinDesc() {
bigTableBucketNumMapping = new LinkedHashMap<String, Integer>();
@@ -123,7 +122,6 @@ public class MapJoinDesc extends JoinDes
}
}
- @Explain(displayName = "input vertices")
public Map<Integer, String> getParentToInput() {
return parentToInput;
}
@@ -333,16 +331,4 @@ public class MapJoinDesc extends JoinDes
public boolean getCustomBucketMapJoin() {
return this.customBucketMapJoin;
}
-
- public boolean isMapSideJoin() {
- return true;
- }
-
- public void setGenJoinKeys(boolean genJoinKeys) {
- this.genJoinKeys = genJoinKeys;
- }
-
- public boolean getGenJoinKeys() {
- return genJoinKeys;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Mon Oct 6 03:44:13 2014
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import com.google.common.collect.Interner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -97,7 +96,6 @@ public class MapWork extends BaseWork {
private Long minSplitSize;
private Long minSplitSizePerNode;
private Long minSplitSizePerRack;
- private final int tag = 0;
//use sampled partitioning
private int samplingType;
@@ -128,8 +126,6 @@ public class MapWork extends BaseWork {
private Map<String, List<ExprNodeDesc>> eventSourcePartKeyExprMap =
new LinkedHashMap<String, List<ExprNodeDesc>>();
- private boolean doSplitsGrouping = true;
-
public MapWork() {}
public MapWork(String name) {
@@ -199,22 +195,6 @@ public class MapWork extends BaseWork {
}
}
- public void internTable(Interner<TableDesc> interner) {
- if (aliasToPartnInfo != null) {
- for (PartitionDesc part : aliasToPartnInfo.values()) {
- if (part == null) {
- continue;
- }
- part.intern(interner);
- }
- }
- if (pathToPartitionInfo != null) {
- for (PartitionDesc part : pathToPartitionInfo.values()) {
- part.intern(interner);
- }
- }
- }
-
/**
* @return the aliasToPartnInfo
*/
@@ -587,12 +567,4 @@ public class MapWork extends BaseWork {
public void setEventSourcePartKeyExprMap(Map<String, List<ExprNodeDesc>> map) {
this.eventSourcePartKeyExprMap = map;
}
-
- public void setDoSplitsGrouping(boolean doSplitsGrouping) {
- this.doSplitsGrouping = doSplitsGrouping;
- }
-
- public boolean getDoSplitsGrouping() {
- return this.doSplitsGrouping;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Mon Oct 6 03:44:13 2014
@@ -20,16 +20,17 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.List;
-public class OpTraits {
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+public class OpTraits {
+
List<List<String>> bucketColNames;
- List<List<String>> sortColNames;
int numBuckets;
-
- public OpTraits(List<List<String>> bucketColNames, int numBuckets, List<List<String>> sortColNames) {
+
+ public OpTraits(List<List<String>> bucketColNames, int numBuckets) {
this.bucketColNames = bucketColNames;
this.numBuckets = numBuckets;
- this.sortColNames = sortColNames;
}
public List<List<String>> getBucketColNames() {
@@ -41,18 +42,10 @@ public class OpTraits {
}
public void setBucketColNames(List<List<String>> bucketColNames) {
- this.bucketColNames = bucketColNames;
+ this.bucketColNames = bucketColNames;
}
public void setNumBuckets(int numBuckets) {
this.numBuckets = numBuckets;
}
-
- public void setSortColNames(List<List<String>> sortColNames) {
- this.sortColNames = sortColNames;
- }
-
- public List<List<String>> getSortCols() {
- return sortColNames;
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Mon Oct 6 03:44:13 2014
@@ -48,10 +48,12 @@ import org.apache.hadoop.util.Reflection
public class PartitionDesc implements Serializable, Cloneable {
static {
+ TABLE_INTERNER = Interners.newWeakInterner();
STRING_INTERNER = Interners.newWeakInterner();
CLASS_INTERNER = Interners.newWeakInterner();
}
+ private static final Interner<TableDesc> TABLE_INTERNER;
private static final Interner<String> STRING_INTERNER;
private static final Interner<Class<?>> CLASS_INTERNER;
@@ -71,12 +73,12 @@ public class PartitionDesc implements Se
}
public PartitionDesc(final TableDesc table, final LinkedHashMap<String, String> partSpec) {
- this.tableDesc = table;
+ setTableDesc(table);
this.partSpec = partSpec;
}
public PartitionDesc(final Partition part) throws HiveException {
- this.tableDesc = Utilities.getTableDesc(part.getTable());
+ setTableDesc(Utilities.getTableDesc(part.getTable()));
setProperties(part.getMetadataFromPartitionSchema());
partSpec = part.getSpec();
setInputFileFormatClass(part.getInputFormatClass());
@@ -84,7 +86,7 @@ public class PartitionDesc implements Se
}
public PartitionDesc(final Partition part,final TableDesc tblDesc) throws HiveException {
- this.tableDesc = tblDesc;
+ setTableDesc(tblDesc);
setProperties(part.getSchemaFromTableSchema(tblDesc.getProperties())); // each partition maintains a large properties
partSpec = part.getSpec();
setOutputFileFormatClass(part.getInputFormatClass());
@@ -97,7 +99,7 @@ public class PartitionDesc implements Se
}
public void setTableDesc(TableDesc tableDesc) {
- this.tableDesc = tableDesc;
+ this.tableDesc = TABLE_INTERNER.intern(tableDesc);
}
@Explain(displayName = "partition values")
@@ -264,8 +266,4 @@ public class PartitionDesc implements Se
baseFileName = path;
}
}
-
- public void intern(Interner<TableDesc> interner) {
- this.tableDesc = interner.intern(tableDesc);
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java Mon Oct 6 03:44:13 2014
@@ -19,13 +19,12 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.LinkedList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -47,22 +46,6 @@ import org.apache.hadoop.mapred.JobConf;
@Explain(displayName = "Tez")
public class TezWork extends AbstractOperatorDesc {
- public enum VertexType {
- AUTO_INITIALIZED_EDGES, // no custom vertex or edge
- INITIALIZED_EDGES, // custom vertex and custom edge but single MR Input
- MULTI_INPUT_INITIALIZED_EDGES, // custom vertex, custom edge and multi MR Input
- MULTI_INPUT_UNINITIALIZED_EDGES // custom vertex, no custom edge, multi MR Input
- ;
-
- public static boolean isCustomInputType(VertexType vertex) {
- if ((vertex == null) || (vertex == AUTO_INITIALIZED_EDGES)) {
- return false;
- } else {
- return true;
- }
- }
- }
-
private static transient final Log LOG = LogFactory.getLog(TezWork.class);
private static int counter;
@@ -73,7 +56,6 @@ public class TezWork extends AbstractOpe
private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork, List<BaseWork>>();
private final Map<Pair<BaseWork, BaseWork>, TezEdgeProperty> edgeProperties =
new HashMap<Pair<BaseWork, BaseWork>, TezEdgeProperty>();
- private final Map<BaseWork, VertexType> workVertexTypeMap = new HashMap<BaseWork, VertexType>();
public TezWork(String name) {
this.name = name + ":" + (++counter);
@@ -323,23 +305,15 @@ public class TezWork extends AbstractOpe
work.configureJobConf(jobConf);
}
String[] newTmpJars = jobConf.getStrings(MR_JAR_PROPERTY);
- if (oldTmpJars != null || newTmpJars != null) {
- String[] finalTmpJars;
- if (oldTmpJars == null || oldTmpJars.length == 0) {
- // Avoid a copy when oldTmpJars is null or empty
- finalTmpJars = newTmpJars;
- } else if (newTmpJars == null || newTmpJars.length == 0) {
- // Avoid a copy when newTmpJars is null or empty
- finalTmpJars = oldTmpJars;
+ if (oldTmpJars != null && (oldTmpJars.length != 0)) {
+ if (newTmpJars != null && (newTmpJars.length != 0)) {
+ String[] combinedTmpJars = new String[newTmpJars.length + oldTmpJars.length];
+ System.arraycopy(oldTmpJars, 0, combinedTmpJars, 0, oldTmpJars.length);
+ System.arraycopy(newTmpJars, 0, combinedTmpJars, oldTmpJars.length, newTmpJars.length);
+ jobConf.setStrings(MR_JAR_PROPERTY, combinedTmpJars);
} else {
- // Both are non-empty, only copy now
- finalTmpJars = new String[oldTmpJars.length + newTmpJars.length];
- System.arraycopy(oldTmpJars, 0, finalTmpJars, 0, oldTmpJars.length);
- System.arraycopy(newTmpJars, 0, finalTmpJars, oldTmpJars.length, newTmpJars.length);
+ jobConf.setStrings(MR_JAR_PROPERTY, oldTmpJars);
}
-
- jobConf.setStrings(MR_JAR_PROPERTY, finalTmpJars);
- return finalTmpJars;
}
return newTmpJars;
}
@@ -358,40 +332,4 @@ public class TezWork extends AbstractOpe
ImmutablePair workPair = new ImmutablePair(a, b);
edgeProperties.put(workPair, edgeProp);
}
-
- public void setVertexType(BaseWork w, VertexType incomingVertexType) {
- VertexType vertexType = workVertexTypeMap.get(w);
- if (vertexType == null) {
- vertexType = VertexType.AUTO_INITIALIZED_EDGES;
- }
- switch (vertexType) {
- case INITIALIZED_EDGES:
- if (incomingVertexType == VertexType.MULTI_INPUT_UNINITIALIZED_EDGES) {
- vertexType = VertexType.MULTI_INPUT_INITIALIZED_EDGES;
- }
- break;
-
- case MULTI_INPUT_INITIALIZED_EDGES:
- // nothing to do
- break;
-
- case MULTI_INPUT_UNINITIALIZED_EDGES:
- if (incomingVertexType == VertexType.INITIALIZED_EDGES) {
- vertexType = VertexType.MULTI_INPUT_INITIALIZED_EDGES;
- }
- break;
-
- case AUTO_INITIALIZED_EDGES:
- vertexType = incomingVertexType;
- break;
-
- default:
- break;
- }
- workVertexTypeMap.put(w, vertexType);
- }
-
- public VertexType getVertexType(BaseWork w) {
- return workVertexTypeMap.get(w);
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java Mon Oct 6 03:44:13 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
* an error while performing authorization, and not a authorization being
* denied.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HiveAccessControlException extends HiveException{
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java Mon Oct 6 03:44:13 2014
@@ -19,7 +19,8 @@ package org.apache.hadoop.hive.ql.securi
import java.util.List;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
import org.apache.hadoop.hive.conf.HiveConf;
/**
@@ -27,7 +28,8 @@ import org.apache.hadoop.hive.conf.HiveC
* create/drop roles, and commands to read the state of authorization rules.
* Methods here have corresponding methods in HiveAuthorizer, check method documentation there.
*/
-@Private
+@LimitedPrivate(value = { "" })
+@Evolving
public interface HiveAccessController {
void grantPrivileges(List<HivePrincipal> hivePrincipals, List<HivePrivilege> hivePrivileges,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java Mon Oct 6 03:44:13 2014
@@ -19,13 +19,15 @@ package org.apache.hadoop.hive.ql.securi
import java.util.List;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
/**
* Interface used to check if user has privileges to perform certain action.
* Methods here have corresponding methods in HiveAuthorizer, check method documentation there.
*/
-@Private
+@LimitedPrivate(value = { "" })
+@Evolving
public interface HiveAuthorizationValidator {
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java Mon Oct 6 03:44:13 2014
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.securit
* statements and does not make assumptions about the privileges needed for a hive operation.
* This is referred to as V2 authorizer in other parts of the code.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public interface HiveAuthorizer {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java Mon Oct 6 03:44:13 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.securit
* create {@link HiveAuthorizer} instance used for hive authorization.
*
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public interface HiveAuthorizerFactory {
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java Mon Oct 6 03:44:13 2014
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.common.cla
* auditing and/or authorization.
* It is an immutable class. Builder inner class is used instantiate it.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public final class HiveAuthzContext {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java Mon Oct 6 03:44:13 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
* an error while performing authorization, and not a authorization being
* denied.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HiveAuthzPluginException extends HiveException{
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java Mon Oct 6 03:44:13 2014
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.common.cla
* Provides session context information.
* It is an immutable class. Builder inner class is used instantiate it.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public final class HiveAuthzSessionContext {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java Mon Oct 6 03:44:13 2014
@@ -28,7 +28,7 @@ import org.apache.hadoop.hive.metastore.
* But Hive class is not a public interface, so this factory helps in hiding Hive
* class from the authorization interface users.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public interface HiveMetastoreClientFactory {
IMetaStoreClient getHiveMetastoreClient() throws HiveAuthzPluginException;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java Mon Oct 6 03:44:13 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
/**
* List of hive operations types.
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public enum HiveOperationType {
EXPLAIN,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java Mon Oct 6 03:44:13 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
/**
* Represents the user or role in grant/revoke statements
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HivePrincipal implements Comparable<HivePrincipal> {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java Mon Oct 6 03:44:13 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.securit
/**
* Represents the hive privilege being granted/revoked
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HivePrivilege implements Comparable<HivePrivilege> {
@Override
@@ -97,7 +97,6 @@ public class HivePrivilege implements Co
return supportedScope != null && supportedScope.contains(scope.name());
}
- @Override
public int compareTo(HivePrivilege privilege) {
int compare = columns != null ?
(privilege.columns != null ? compare(columns, privilege.columns) : 1) :
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java Mon Oct 6 03:44:13 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
/**
* Represents a privilege granted for an object to a principal
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HivePrivilegeInfo{
private final HivePrincipal principal;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java Mon Oct 6 03:44:13 2014
@@ -22,19 +22,14 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
/**
- * Represents the object on which privilege is being granted/revoked, and objects
- * being used in queries.
- *
- * Check the get* function documentation for information on what value it returns based on
- * the {@link HivePrivilegeObjectType}.
- *
+ * Represents the object on which privilege is being granted/revoked
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
-@Evolving
+@LimitedPrivate(value = { "" })
+@Unstable
public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> {
@Override
@@ -82,20 +77,9 @@ public class HivePrivilegeObject impleme
return o1.size() > o2.size() ? 1 : (o1.size() < o2.size() ? -1 : 0);
}
- /**
- * Note that GLOBAL, PARTITION, COLUMN fields are populated only for Hive's old default
- * authorization mode.
- * When the authorization manager is an instance of HiveAuthorizerFactory, these types are not
- * used.
- */
public enum HivePrivilegeObjectType {
GLOBAL, DATABASE, TABLE_OR_VIEW, PARTITION, COLUMN, LOCAL_URI, DFS_URI, COMMAND_PARAMS, FUNCTION
- };
-
- /**
- * When {@link HiveOperationType} is QUERY, this action type is set so that it is possible
- * to determine if the action type on this object is an INSERT or INSERT_OVERWRITE
- */
+ } ;
public enum HivePrivObjectActionType {
OTHER, INSERT, INSERT_OVERWRITE, UPDATE, DELETE
};
@@ -155,9 +139,6 @@ public class HivePrivilegeObject impleme
return type;
}
- /**
- * @return the db name if type is DATABASE, TABLE, or FUNCTION
- */
public String getDbname() {
return dbname;
}
@@ -169,10 +150,6 @@ public class HivePrivilegeObject impleme
return objectName;
}
- /**
- * See javadoc of {@link HivePrivObjectActionType}
- * @return action type
- */
public HivePrivObjectActionType getActionType() {
return actionType;
}
@@ -181,15 +158,12 @@ public class HivePrivilegeObject impleme
return commandParams;
}
- /**
- * @return partiton key information. Used only for old default authorization mode.
- */
public List<String> getPartKeys() {
return partKeys;
}
/**
- * Applicable columns in this object, when the type is {@link HivePrivilegeObjectType.TABLE}
+ * Applicable columns in this object
* In case of DML read operations, this is the set of columns being used.
* Column information is not set for DDL operations and for tables being written into
* @return list of applicable columns
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java Mon Oct 6 03:44:13 2014
@@ -26,7 +26,7 @@ import com.google.common.collect.Compari
/**
* Represents a grant of a role to a principal
*/
-@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@LimitedPrivate(value = { "" })
@Evolving
public class HiveRoleGrant implements Comparable<HiveRoleGrant> {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Mon Oct 6 03:44:13 2014
@@ -1249,7 +1249,7 @@ public class SessionState {
try {
if (tezSessionState != null) {
- TezSessionPoolManager.getInstance().close(tezSessionState, false);
+ TezSessionPoolManager.getInstance().close(tezSessionState);
}
} catch (Exception e) {
LOG.info("Error closing tez session", e);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Mon Oct 6 03:44:13 2014
@@ -18,15 +18,8 @@
package org.apache.hadoop.hive.ql.stats;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -87,14 +80,19 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
import org.apache.hadoop.io.BytesWritable;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class StatsUtils {
private static final Log LOG = LogFactory.getLog(StatsUtils.class.getName());
-
/**
* Collect table, partition and column level statistics
* @param conf
@@ -111,34 +109,15 @@ public class StatsUtils {
public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
Table table, TableScanOperator tableScanOperator) throws HiveException {
+ Statistics stats = new Statistics();
+
// column level statistics are required only for the columns that are needed
List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature();
List<String> neededColumns = tableScanOperator.getNeededColumns();
- List<String> referencedColumns = tableScanOperator.getReferencedColumns();
-
- return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns);
- }
-
- private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
- Table table, List<ColumnInfo> schema, List<String> neededColumns,
- List<String> referencedColumns) throws HiveException {
-
boolean fetchColStats =
HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_COLUMN_STATS);
boolean fetchPartStats =
HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_PARTITION_STATS);
-
- return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns,
- fetchColStats, fetchPartStats);
- }
-
- public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
- Table table, List<ColumnInfo> schema, List<String> neededColumns,
- List<String> referencedColumns, boolean fetchColStats, boolean fetchPartStats)
- throws HiveException {
-
- Statistics stats = new Statistics();
-
float deserFactor =
HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR);
@@ -228,6 +207,7 @@ public class StatsUtils {
stats.getBasicStatsState().equals(State.COMPLETE)) {
stats.setBasicStatsState(State.PARTIAL);
}
+ boolean haveFullStats = fetchColStats;
if (fetchColStats) {
List<String> partNames = new ArrayList<String>(partList.getNotDeniedPartns().size());
for (Partition part : partList.getNotDeniedPartns()) {
@@ -235,82 +215,35 @@ public class StatsUtils {
}
Map<String, String> colToTabAlias = new HashMap<String, String>();
neededColumns = processNeededColumns(schema, neededColumns, colToTabAlias);
- AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
- neededColumns, partNames);
+ AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(), neededColumns, partNames);
if (null == aggrStats) {
- // There are some partitions with no state (or we didn't fetch any state).
- // Update the stats with empty list to reflect that in the
- // state/initialize structures.
- List<ColStatistics> emptyStats = Lists.newArrayList();
-
- // add partition column stats
- addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList,
- emptyStats);
-
- stats.addToColumnStats(emptyStats);
- stats.updateColumnStatsState(deriveStatType(emptyStats, referencedColumns));
+ haveFullStats = false;
} else {
List<ColumnStatisticsObj> colStats = aggrStats.getColStats();
if (colStats.size() != neededColumns.size()) {
- LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to" +
- " retrieve for " + colStats.size() + " columns");
+ LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to retrieve"
+ + " for " + colStats.size() + " columns");
}
- List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(),
- colToTabAlias);
-
- addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList,
- columnStats);
-
+ List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(), colToTabAlias);
stats.addToColumnStats(columnStats);
- State colState = deriveStatType(columnStats, referencedColumns);
+ State colState = deriveStatType(columnStats, neededColumns);
if (aggrStats.getPartsFound() != partNames.size() && colState != State.NONE) {
- LOG.debug("Column stats requested for : " + partNames.size() + " partitions. "
- + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions");
+ LOG.debug("Column stats requested for : " + partNames.size() +" partitions. "
+ + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions");
colState = State.PARTIAL;
}
stats.setColumnStatsState(colState);
}
}
- }
- return stats;
- }
-
- private static void addParitionColumnStats(List<String> neededColumns,
- List<String> referencedColumns, List<ColumnInfo> schema, Table table,
- PrunedPartitionList partList, List<ColStatistics> colStats)
- throws HiveException {
-
- // extra columns is difference between referenced columns vs needed
- // columns. The difference could be partition columns.
- List<String> extraCols = Lists.newArrayList(referencedColumns);
- if (referencedColumns.size() > neededColumns.size()) {
- extraCols.removeAll(neededColumns);
- for (String col : extraCols) {
- for (ColumnInfo ci : schema) {
- // conditions for being partition column
- if (col.equals(ci.getInternalName()) && ci.getIsVirtualCol() &&
- !ci.isHiddenVirtualCol()) {
- // currently metastore does not store column stats for
- // partition column, so we calculate the NDV from pruned
- // partition list
- ColStatistics partCS = new ColStatistics(table.getTableName(),
- ci.getInternalName(), ci.getType().getTypeName());
- long numPartitions = getNDVPartitionColumn(partList.getPartitions(),
- ci.getInternalName());
- partCS.setCountDistint(numPartitions);
- colStats.add(partCS);
- }
- }
+ // There are some partitions with no state (or we didn't fetch any state).
+ // Update the stats with empty list to reflect that in the state/initialize structures.
+ if (!haveFullStats) {
+ List<ColStatistics> emptyStats = Lists.<ColStatistics>newArrayList();
+ stats.addToColumnStats(emptyStats);
+ stats.updateColumnStatsState(deriveStatType(emptyStats, neededColumns));
}
}
- }
-
- public static int getNDVPartitionColumn(Set<Partition> partitions, String partColName) {
- Set<String> distinctVals = new HashSet<String>(partitions.size());
- for (Partition partition : partitions) {
- distinctVals.add(partition.getSpec().get(partColName));
- }
- return distinctVals.size();
+ return stats;
}
private static void setUnknownRcDsToAverage(
@@ -818,8 +751,7 @@ public class StatsUtils {
|| colType.equalsIgnoreCase(serdeConstants.FLOAT_TYPE_NAME)) {
return JavaDataModel.get().primitive1();
} else if (colType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)
- || colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)
- || colType.equalsIgnoreCase("long")) {
+ || colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) {
return JavaDataModel.get().primitive2();
} else if (colType.equalsIgnoreCase(serdeConstants.TIMESTAMP_TYPE_NAME)) {
return JavaDataModel.get().lengthOfTimestamp();
@@ -848,8 +780,7 @@ public class StatsUtils {
return JavaDataModel.get().lengthForIntArrayOfSize(length);
} else if (colType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)) {
return JavaDataModel.get().lengthForDoubleArrayOfSize(length);
- } else if (colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)
- || colType.equalsIgnoreCase("long")) {
+ } else if (colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) {
return JavaDataModel.get().lengthForLongArrayOfSize(length);
} else if (colType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)) {
return JavaDataModel.get().lengthForByteArrayOfSize(length);
@@ -945,7 +876,7 @@ public class StatsUtils {
Statistics parentStats, Map<String, ExprNodeDesc> colExprMap, RowSchema rowSchema) {
List<ColStatistics> cs = Lists.newArrayList();
- if (colExprMap != null && rowSchema != null) {
+ if (colExprMap != null) {
for (ColumnInfo ci : rowSchema.getSignature()) {
String outColName = ci.getInternalName();
outColName = StatsUtils.stripPrefixFromColumnName(outColName);
@@ -1111,8 +1042,10 @@ public class StatsUtils {
/**
* Get basic stats of table
- * @param table
- * - table
+ * @param dbName
+ * - database name
+ * @param tabName
+ * - table name
* @param statType
* - type of stats
* @return value of stats
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Mon Oct 6 03:44:13 2014
@@ -76,7 +76,7 @@ public class Initiator extends Compactor
// don't doom the entire thread.
try {
ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest());
- ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
+ ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold);
LOG.debug("Found " + potentials.size() + " potential compactions, " +
"checking to see if we should compact any of them");
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Mon Oct 6 03:44:13 2014
@@ -120,7 +120,7 @@ public class Worker extends CompactorThr
final boolean isMajor = ci.isMajorCompaction();
final ValidTxnList txns =
- TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
+ TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
final StringBuffer jobName = new StringBuffer(name);
jobName.append("-compactor-");
jobName.append(ci.getFullPartitionName());
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java Mon Oct 6 03:44:13 2014
@@ -52,6 +52,29 @@ public class UDFLog extends UDFMath {
}
/**
+ * Get the logarithm of the given decimal with the given base.
+ */
+ public DoubleWritable evaluate(DoubleWritable base, HiveDecimalWritable writable) {
+ if (base == null || writable == null) {
+ return null;
+ }
+ double d = writable.getHiveDecimal().bigDecimalValue().doubleValue();
+ return log(base.get(), d);
+ }
+
+ /**
+ * Get the logarithm of input with the given decimal as the base.
+ */
+ public DoubleWritable evaluate(HiveDecimalWritable base, DoubleWritable d) {
+ if (base == null || d == null) {
+ return null;
+ }
+
+ double b = base.getHiveDecimal().bigDecimalValue().doubleValue();
+ return log(b, d.get());
+ }
+
+ /**
* Get the logarithm of the given decimal input with the given decimal base.
*/
public DoubleWritable evaluate(HiveDecimalWritable baseWritable, HiveDecimalWritable writable) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java Mon Oct 6 03:44:13 2014
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.ql.exec.No
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
@@ -45,7 +44,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -320,17 +318,4 @@ public abstract class GenericUDFBaseNume
public void setAnsiSqlArithmetic(boolean ansiSqlArithmetic) {
this.ansiSqlArithmetic = ansiSqlArithmetic;
}
-
- public PrimitiveTypeInfo deriveMinArgumentCast(
- ExprNodeDesc childExpr, TypeInfo targetType) {
- assert targetType instanceof PrimitiveTypeInfo : "Not a primitive type" + targetType;
- PrimitiveTypeInfo pti = (PrimitiveTypeInfo)targetType;
- // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
- // We also don't do anything for non-primitive children (maybe we should assert).
- if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
- || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
- PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
- // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
- return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
- }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java Mon Oct 6 03:44:13 2014
@@ -22,7 +22,6 @@ import java.util.TimeZone;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -34,9 +33,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Text;
-@Description(name = "from_utc_timestamp",
- value = "from_utc_timestamp(timestamp, string timezone) - "
- + "Assumes given timestamp ist UTC and converts to given timezone (as of Hive 0.8.0)")
+
public class GenericUDFFromUtcTimestamp extends GenericUDF {
static final Log LOG = LogFactory.getLog(GenericUDFFromUtcTimestamp.class);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java Mon Oct 6 03:44:13 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.udf.generic;
-import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -63,11 +62,6 @@ import org.apache.hadoop.hive.ql.exec.ve
* otherwise it returns expr3. IF() returns a numeric or string value, depending
* on the context in which it is used.
*/
-@Description(
- name = "if",
- value = "IF(expr1,expr2,expr3) - If expr1 is TRUE (expr1 <> 0 and expr1 <> NULL) then"
- + " IF() returns expr2; otherwise it returns expr3. IF() returns a numeric or string value,"
- + " depending on the context in which it is used.")
@VectorizedExpressions({
IfExprLongColumnLongColumn.class, IfExprDoubleColumnDoubleColumn.class,
IfExprLongColumnLongScalar.class, IfExprDoubleColumnDoubleScalar.class,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java Mon Oct 6 03:44:13 2014
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
-import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
@@ -40,8 +39,6 @@ import org.apache.hadoop.hive.serde2.obj
* Creates a TimestampWritable object using PrimitiveObjectInspectorConverter
*
*/
-@Description(name = "timestamp",
-value = "cast(date as timestamp) - Returns timestamp")
@VectorizedExpressions({CastLongToTimestampViaLongToLong.class,
CastDoubleToTimestampViaDoubleToLong.class, CastDecimalToTimestamp.class})
public class GenericUDFTimestamp extends GenericUDF {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java?rev=1629562&r1=1629561&r2=1629562&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java Mon Oct 6 03:44:13 2014
@@ -17,11 +17,7 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
-import org.apache.hadoop.hive.ql.exec.Description;
-@Description(name = "to_utc_timestamp",
- value = "to_utc_timestamp(timestamp, string timezone) - "
- + "Assumes given timestamp is in given timezone and converts to UTC (as of Hive 0.8.0)")
public class GenericUDFToUtcTimestamp extends
GenericUDFFromUtcTimestamp {