You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/04/15 01:36:09 UTC
svn commit: r1673583 [9/27] - in /hive/branches/spark: ./
beeline/src/java/org/apache/hive/beeline/ bin/
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/ common/
common/src/java/org/apache/hadoop/hive/common/ common/src...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Apr 14 23:36:02 2015
@@ -196,18 +196,13 @@ public class PartitionPruner implements
// Remove all parts that are not partition columns. See javadoc for details.
ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
String oldFilter = prunerExpr.getExprString();
- if (isBooleanExpr(compactExpr)) {
- // For null and true values, return every partition
- if (!isFalseExpr(compactExpr)) {
- // Non-strict mode, and all the predicates are on non-partition columns - get everything.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Filter " + oldFilter + " was null after compacting");
- }
- return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
- } else {
- return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(new ArrayList<Partition>()),
- new ArrayList<String>(), false);
- }
+ if (compactExpr == null || isBooleanExpr(compactExpr)) {
+ if (isFalseExpr(compactExpr)) {
+ return new PrunedPartitionList(
+ tab, new LinkedHashSet<Partition>(0), new ArrayList<String>(0), false);
+ }
+ // For null and true values, return every partition
+ return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
@@ -241,22 +236,22 @@ public class PartitionPruner implements
partsCache.put(key, ppList);
return ppList;
}
-
+
static private boolean isBooleanExpr(ExprNodeDesc expr) {
- return expr != null && expr instanceof ExprNodeConstantDesc &&
+ return expr != null && expr instanceof ExprNodeConstantDesc &&
((ExprNodeConstantDesc)expr).getTypeInfo() instanceof PrimitiveTypeInfo &&
((PrimitiveTypeInfo)(((ExprNodeConstantDesc)expr).getTypeInfo())).
- getTypeName().equals(serdeConstants.BOOLEAN_TYPE_NAME);
+ getTypeName().equals(serdeConstants.BOOLEAN_TYPE_NAME);
}
static private boolean isTrueExpr(ExprNodeDesc expr) {
- return isBooleanExpr(expr) &&
- ((ExprNodeConstantDesc)expr).getValue() != null &&
- ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.TRUE);
+ return isBooleanExpr(expr) &&
+ ((ExprNodeConstantDesc)expr).getValue() != null &&
+ ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.TRUE);
}
static private boolean isFalseExpr(ExprNodeDesc expr) {
- return isBooleanExpr(expr) &&
+ return isBooleanExpr(expr) &&
((ExprNodeConstantDesc)expr).getValue() != null &&
- ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.FALSE);
+ ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.FALSE);
}
/**
@@ -268,42 +263,48 @@ public class PartitionPruner implements
*/
static private ExprNodeDesc compactExpr(ExprNodeDesc expr) {
// If this is a constant boolean expression, return the value.
- if (expr == null) {
- return null;
- }
- if (expr instanceof ExprNodeConstantDesc) {
- if (isBooleanExpr(expr)) {
- return expr;
- } else {
- throw new IllegalStateException("Unexpected non-null ExprNodeConstantDesc: "
- + expr.getExprString());
+ if (expr == null) {
+ return null;
+ }
+ if (expr instanceof ExprNodeConstantDesc) {
+ if (((ExprNodeConstantDesc)expr).getValue() == null) return null;
+ if (!isBooleanExpr(expr)) {
+ throw new IllegalStateException("Unexpected non-boolean ExprNodeConstantDesc: "
+ + expr.getExprString());
}
+ return expr;
} else if (expr instanceof ExprNodeGenericFuncDesc) {
GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
boolean isAnd = udf instanceof GenericUDFOPAnd;
boolean isOr = udf instanceof GenericUDFOPOr;
-
+
if (isAnd || isOr) {
List<ExprNodeDesc> children = expr.getChildren();
- ExprNodeDesc left = children.get(0);
- children.set(0, compactExpr(left));
- ExprNodeDesc right = children.get(1);
- children.set(1, compactExpr(right));
-
- if (isTrueExpr(children.get(0)) && isTrueExpr(children.get(1))) {
- return new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isTrueExpr(children.get(0))) {
- return isAnd ? children.get(1) : new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isTrueExpr(children.get(1))) {
- return isAnd ? children.get(0) : new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isFalseExpr(children.get(0)) && isFalseExpr(children.get(1))) {
- return new ExprNodeConstantDesc(Boolean.FALSE);
- } else if (isFalseExpr(children.get(0))) {
- return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(1);
- } else if (isFalseExpr(children.get(1))) {
- return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(0);
- }
-
+ ExprNodeDesc left = compactExpr(children.get(0));
+ ExprNodeDesc right = compactExpr(children.get(1));
+ // Non-partition expressions are converted to nulls.
+ if (left == null && right == null) {
+ return null;
+ } else if (left == null) {
+ return isAnd ? right : null;
+ } else if (right == null) {
+ return isAnd ? left : null;
+ }
+ // Handle boolean expressions
+ boolean isLeftFalse = isFalseExpr(left), isRightFalse = isFalseExpr(right),
+ isLeftTrue = isTrueExpr(left), isRightTrue = isTrueExpr(right);
+ if ((isRightTrue && isLeftTrue) || (isOr && (isLeftTrue || isRightTrue))) {
+ return new ExprNodeConstantDesc(Boolean.TRUE);
+ } else if ((isRightFalse && isLeftFalse) || (isAnd && (isLeftFalse || isRightFalse))) {
+ return new ExprNodeConstantDesc(Boolean.FALSE);
+ } else if ((isAnd && isLeftTrue) || (isOr && isLeftFalse)) {
+ return right;
+ } else if ((isAnd && isRightTrue) || (isOr && isRightFalse)) {
+ return left;
+ }
+ // Nothing to compact, update expr with compacted children.
+ children.set(0, left);
+ children.set(1, right);
}
return expr;
} else {
@@ -328,9 +329,9 @@ public class PartitionPruner implements
if (!partCols.contains(column)) {
// Column doesn't appear to be a partition column for the table.
return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
- }
+ }
referred.add(column);
- }
+ }
if (expr instanceof ExprNodeGenericFuncDesc) {
List<ExprNodeDesc> children = expr.getChildren();
for (int i = 0; i < children.size(); ++i) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java Tue Apr 14 23:36:02 2015
@@ -103,7 +103,7 @@ public class SparkMapJoinOptimizer imple
}
// we can set the traits for this join operator
- OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks());
+ OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null);
mapJoinOp.setOpTraits(opTraits);
mapJoinOp.setStatistics(joinOp.getStatistics());
setNumberOfBucketsOnChildren(mapJoinOp);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java Tue Apr 14 23:36:02 2015
@@ -26,8 +26,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
-@Explain(displayName = "Alter Table Partition Merge Files")
+
+@Explain(displayName = "Alter Table Partition Merge Files", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class AlterTablePartMergeFilesDesc {
private String tableName;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Apr 14 23:36:02 2015
@@ -69,6 +69,7 @@ import org.apache.calcite.rel.rules.Filt
import org.apache.calcite.rel.rules.JoinPushTransitivePredicatesRule;
import org.apache.calcite.rel.rules.JoinToMultiJoinRule;
import org.apache.calcite.rel.rules.LoptOptimizeJoinRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
import org.apache.calcite.rel.rules.ProjectRemoveRule;
import org.apache.calcite.rel.rules.ReduceExpressionsRule;
import org.apache.calcite.rel.rules.SemiJoinFilterTransposeRule;
@@ -120,6 +121,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveVolcanoPlanner;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
@@ -248,13 +250,29 @@ public class CalcitePlanner extends Sema
disableJoinMerge = false;
sinkOp = genPlan(getQB());
LOG.info("CBO Succeeded; optimized logical plan.");
+ this.ctx.setCboInfo("Plan optimized by CBO.");
LOG.debug(newAST.dump());
} catch (Exception e) {
boolean isMissingStats = noColsMissingStats.get() > 0;
if (isMissingStats) {
LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO");
+ this.ctx
+ .setCboInfo("Plan not optimized by CBO due to missing statistics. Please check log for more details.");
} else {
LOG.error("CBO failed, skipping CBO. ", e);
+ if (e instanceof CalciteSemanticException) {
+ CalciteSemanticException calciteSemanticException = (CalciteSemanticException) e;
+ UnsupportedFeature unsupportedFeature = calciteSemanticException
+ .getUnsupportedFeature();
+ if (unsupportedFeature != null) {
+ this.ctx.setCboInfo("Plan not optimized by CBO due to missing feature ["
+ + unsupportedFeature + "].");
+ } else {
+ this.ctx.setCboInfo("Plan not optimized by CBO.");
+ }
+ } else {
+ this.ctx.setCboInfo("Plan not optimized by CBO.");
+ }
}
if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats
|| e instanceof CalciteSemanticException) {
@@ -279,6 +297,7 @@ public class CalcitePlanner extends Sema
}
}
} else {
+ this.ctx.setCboInfo("Plan not optimized by CBO.");
skipCalcitePlan = true;
}
}
@@ -721,6 +740,7 @@ public class CalcitePlanner extends Sema
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
+ hepPgmBldr.addRuleInstance(new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY));
hepPgm = hepPgmBldr.build();
HepPlanner hepPlanner = new HepPlanner(hepPgm);
@@ -785,8 +805,6 @@ public class CalcitePlanner extends Sema
// 3. Transitive inference & Partition Pruning
basePlan = hepPlan(basePlan, false, mdProvider, new JoinPushTransitivePredicatesRule(
Join.class, HiveFilter.DEFAULT_FILTER_FACTORY),
- // TODO: Enable it after CALCITE-407 is fixed
- // RemoveTrivialProjectRule.INSTANCE,
new HivePartitionPruneRule(conf));
// 4. Projection Pruning
@@ -1081,7 +1099,7 @@ public class CalcitePlanner extends Sema
String msg = String.format("UNIQUE JOIN is currently not supported in CBO,"
+ " turn off cbo to use UNIQUE JOIN.");
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Unique_join);
}
// 1. Determine Join Type
@@ -1164,7 +1182,7 @@ public class CalcitePlanner extends Sema
+ " Currently we don't support Table Sample clauses in CBO,"
+ " turn off cbo for queries on tableSamples.", tableAlias);
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Table_sample_clauses);
}
// 2. Get Table Metadata
@@ -1261,7 +1279,8 @@ public class CalcitePlanner extends Sema
// fail on compile time
// for such queries, its an arcane corner case, not worth of adding that
// complexity.
- throw new CalciteSemanticException("Filter expression with non-boolean return type.");
+ throw new CalciteSemanticException("Filter expression with non-boolean return type.",
+ UnsupportedFeature.Filter_expression_with_non_boolean_return_type);
}
ImmutableMap<String, Integer> hiveColNameCalcitePosMap = this.relToHiveColNameCalcitePosMap
.get(srcRel);
@@ -1772,7 +1791,8 @@ public class CalcitePlanner extends Sema
grpbyExpr, new TypeCheckCtx(groupByInputRowResolver));
ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr);
if (grpbyExprNDesc == null)
- throw new CalciteSemanticException("Invalid Column Reference: " + grpbyExpr.dump());
+ throw new CalciteSemanticException("Invalid Column Reference: " + grpbyExpr.dump(),
+ UnsupportedFeature.Invalid_column_reference);
addToGBExpr(groupByOutputRowResolver, groupByInputRowResolver, grpbyExpr,
grpbyExprNDesc, gbExprNDescLst, outputColumnNames);
@@ -1960,7 +1980,8 @@ public class CalcitePlanner extends Sema
RowResolver obSyntheticProjectRR = new RowResolver();
if (!RowResolver.add(obSyntheticProjectRR, inputRR)) {
throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message");
+ "Duplicates detected when adding columns to RR: see previous message",
+ UnsupportedFeature.Duplicates_in_RR);
}
int vcolPos = inputRR.getRowSchema().getSignature().size();
for (Pair<ASTNode, TypeInfo> astTypePair : vcASTTypePairs) {
@@ -1975,20 +1996,23 @@ public class CalcitePlanner extends Sema
if (outermostOB) {
if (!RowResolver.add(outputRR, inputRR)) {
throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message");
+ "Duplicates detected when adding columns to RR: see previous message",
+ UnsupportedFeature.Duplicates_in_RR);
}
} else {
if (!RowResolver.add(outputRR, obSyntheticProjectRR)) {
throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message");
+ "Duplicates detected when adding columns to RR: see previous message",
+ UnsupportedFeature.Duplicates_in_RR);
}
originalOBChild = srcRel;
}
} else {
if (!RowResolver.add(outputRR, inputRR)) {
throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message");
+ "Duplicates detected when adding columns to RR: see previous message",
+ UnsupportedFeature.Duplicates_in_RR);
}
}
@@ -2026,7 +2050,8 @@ public class CalcitePlanner extends Sema
RowResolver outputRR = new RowResolver();
if (!RowResolver.add(outputRR, relToHiveRR.get(srcRel))) {
throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message");
+ "Duplicates detected when adding columns to RR: see previous message",
+ UnsupportedFeature.Duplicates_in_RR);
}
ImmutableMap<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(
outputRR, sortRel);
@@ -2317,7 +2342,7 @@ public class CalcitePlanner extends Sema
String msg = String.format("Hint specified for %s."
+ " Currently we don't support hints in CBO, turn off cbo to use hints.", hint);
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Hint);
}
// 4. Bailout if select involves Transform
@@ -2326,7 +2351,7 @@ public class CalcitePlanner extends Sema
String msg = String.format("SELECT TRANSFORM is currently not supported in CBO,"
+ " turn off cbo to use TRANSFORM.");
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Select_transform);
}
// 5. Bailout if select involves UDTF
@@ -2339,7 +2364,7 @@ public class CalcitePlanner extends Sema
String msg = String.format("UDTF " + funcName + " is currently not supported in CBO,"
+ " turn off cbo to use UDTF " + funcName);
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.UDTF);
}
}
@@ -2408,7 +2433,8 @@ public class CalcitePlanner extends Sema
} else if (expr.toStringTree().contains("TOK_FUNCTIONDI")
&& !(srcRel instanceof HiveAggregate)) {
// Likely a malformed query eg, select hash(distinct c1) from t1;
- throw new CalciteSemanticException("Distinct without an aggreggation.");
+ throw new CalciteSemanticException("Distinct without an aggreggation.",
+ UnsupportedFeature.Distinct_without_an_aggreggation);
} else {
// Case when this is an expression
TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR);
@@ -2427,7 +2453,8 @@ public class CalcitePlanner extends Sema
.isSkewedCol() : false);
if (!out_rwsch.putWithCheck(tabAlias, colAlias, null, colInfo)) {
throw new CalciteSemanticException("Cannot add column to RR: " + tabAlias + "."
- + colAlias + " => " + colInfo + " due to duplication, see previous warnings");
+ + colAlias + " => " + colInfo + " due to duplication, see previous warnings",
+ UnsupportedFeature.Duplicates_in_RR);
}
if (exp instanceof ExprNodeColumnDesc) {
@@ -2491,7 +2518,7 @@ public class CalcitePlanner extends Sema
if (LOG.isDebugEnabled()) {
LOG.debug(msg + " because it: " + reason);
}
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Subquery);
}
// 1. Build Rel For Src (SubQuery, TS, Join)
@@ -2520,7 +2547,7 @@ public class CalcitePlanner extends Sema
// table
// So, for now lets just disable this. Anyway there is nothing much to
// optimize in such cases.
- throw new CalciteSemanticException("Unsupported");
+ throw new CalciteSemanticException("Unsupported", UnsupportedFeature.Others);
}
// 1.3 process join
@@ -2628,7 +2655,8 @@ public class CalcitePlanner extends Sema
if (havingClause != null) {
if (!(srcRel instanceof HiveAggregate)) {
// ill-formed query like select * from t1 having c1 > 0;
- throw new CalciteSemanticException("Having clause without any group-by.");
+ throw new CalciteSemanticException("Having clause without any group-by.",
+ UnsupportedFeature.Having_clause_without_any_groupby);
}
validateNoHavingReferenceToAlias(qb, (ASTNode) havingClause.getChild(0));
gbFilter = genFilterRelNode(qb, (ASTNode) havingClause.getChild(0), srcRel, aliasToRel,
@@ -2690,7 +2718,7 @@ public class CalcitePlanner extends Sema
+ " This non standard behavior is not supported with cbo on."
+ " Turn off cbo for these queries.", aliasToCheck, havingClause);
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Select_alias_in_having_clause);
}
}
@@ -2726,7 +2754,7 @@ public class CalcitePlanner extends Sema
String msg = String.format("Multi Insert is currently not supported in CBO,"
+ " turn off cbo to use Multi Insert.");
LOG.debug(msg);
- throw new CalciteSemanticException(msg);
+ throw new CalciteSemanticException(msg, UnsupportedFeature.Multi_insert);
}
return qbp;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Tue Apr 14 23:36:02 2015
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde.serdeConstants;
/**
* ColumnStatsSemanticAnalyzer.
@@ -186,15 +187,7 @@ public class ColumnStatsSemanticAnalyzer
} else {
whereClause.append(" and ");
}
- whereClause.append(partKey);
- whereClause.append(" = ");
- if (getColTypeOf(partKey).equalsIgnoreCase("string")) {
- whereClause.append("'");
- }
- whereClause.append(value);
- if (getColTypeOf(partKey).equalsIgnoreCase("string")) {
- whereClause.append("'");
- }
+ whereClause.append(partKey).append(" = ").append(genPartValueString(partKey, value));
}
}
@@ -211,11 +204,39 @@ public class ColumnStatsSemanticAnalyzer
return predPresent ? whereClause.append(groupByClause) : groupByClause;
}
+ private String genPartValueString (String partKey, String partVal) throws SemanticException {
+ String returnVal = partVal;
+ String partColType = getColTypeOf(partKey);
+ if (partColType.equals(serdeConstants.STRING_TYPE_NAME) ||
+ partColType.contains(serdeConstants.VARCHAR_TYPE_NAME) ||
+ partColType.contains(serdeConstants.CHAR_TYPE_NAME)) {
+ returnVal = "'" + partVal + "'";
+ } else if (partColType.equals(serdeConstants.TINYINT_TYPE_NAME)) {
+ returnVal = partVal+"Y";
+ } else if (partColType.equals(serdeConstants.SMALLINT_TYPE_NAME)) {
+ returnVal = partVal+"S";
+ } else if (partColType.equals(serdeConstants.INT_TYPE_NAME)) {
+ returnVal = partVal;
+ } else if (partColType.equals(serdeConstants.BIGINT_TYPE_NAME)) {
+ returnVal = partVal+"L";
+ } else if (partColType.contains(serdeConstants.DECIMAL_TYPE_NAME)) {
+ returnVal = partVal + "BD";
+ } else if (partColType.equals(serdeConstants.DATE_TYPE_NAME) ||
+ partColType.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) {
+ returnVal = partColType + " '" + partVal + "'";
+ } else {
+ //for other usually not used types, just quote the value
+ returnVal = "'" + partVal + "'";
+ }
+
+ return returnVal;
+ }
+
private String getColTypeOf (String partKey) throws SemanticException{
for (FieldSchema fs : tbl.getPartitionKeys()) {
if (partKey.equalsIgnoreCase(fs.getName())) {
- return fs.getType();
+ return fs.getType().toLowerCase();
}
}
throw new SemanticException ("Unknown partition key : " + partKey);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Apr 14 23:36:02 2015
@@ -186,6 +186,8 @@ public class DDLSemanticAnalyzer extends
TokenToTypeName.put(HiveParser.TOK_DATE, serdeConstants.DATE_TYPE_NAME);
TokenToTypeName.put(HiveParser.TOK_DATETIME, serdeConstants.DATETIME_TYPE_NAME);
TokenToTypeName.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME);
+ TokenToTypeName.put(HiveParser.TOK_INTERVAL_YEAR_MONTH, serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME);
+ TokenToTypeName.put(HiveParser.TOK_INTERVAL_DAY_TIME, serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME);
TokenToTypeName.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java Tue Apr 14 23:36:02 2015
@@ -91,6 +91,8 @@ public class ExplainSemanticAnalyzer ext
pCtx = ((SemanticAnalyzer)sem).getParseContext();
}
+ boolean userLevelExplain = !extended && !formatted && !dependency && !logical && !authorize
+ && HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_EXPLAIN_USER);
ExplainWork work = new ExplainWork(ctx.getResFile(),
pCtx,
tasks,
@@ -101,7 +103,9 @@ public class ExplainSemanticAnalyzer ext
formatted,
dependency,
logical,
- authorize);
+ authorize,
+ userLevelExplain,
+ ctx.getCboInfo());
work.setAppendTaskType(
HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES));
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java Tue Apr 14 23:36:02 2015
@@ -105,7 +105,7 @@ public class GenTezUtils {
float minPartitionFactor = context.conf.getFloatVar(HiveConf.ConfVars.TEZ_MIN_PARTITION_FACTOR);
long bytesPerReducer = context.conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER);
- ReduceWork reduceWork = new ReduceWork("Reducer "+ (++sequenceNumber));
+ ReduceWork reduceWork = new ReduceWork(Utilities.REDUCENAME + (++sequenceNumber));
LOG.debug("Adding reduce work (" + reduceWork.getName() + ") for " + root);
reduceWork.setReducer(root);
reduceWork.setNeedsTagging(GenMapRedUtils.needsTagging(reduceWork));
@@ -180,7 +180,7 @@ public class GenTezUtils {
public MapWork createMapWork(GenTezProcContext context, Operator<?> root,
TezWork tezWork, PrunedPartitionList partitions) throws SemanticException {
assert root.getParentOperators().isEmpty();
- MapWork mapWork = new MapWork("Map "+ (++sequenceNumber));
+ MapWork mapWork = new MapWork(Utilities.MAPNAME + (++sequenceNumber));
LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root);
// map work starts with table scan operators
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java Tue Apr 14 23:36:02 2015
@@ -166,7 +166,7 @@ public class GenTezWork implements NodeP
}
// connect the work correctly.
work.addSortCols(root.getOpTraits().getSortCols().get(0));
- mergeJoinWork.addMergedWork(work, null);
+ mergeJoinWork.addMergedWork(work, null, context.leafOperatorToFollowingWork);
Operator<? extends OperatorDesc> parentOp =
getParentFromStack(context.currentMergeJoinOperator, stack);
int pos = context.currentMergeJoinOperator.getTagForOperator(parentOp);
@@ -268,6 +268,7 @@ public class GenTezWork implements NodeP
if (LOG.isDebugEnabled()) {
LOG.debug("Removing " + parent + " as parent from " + root);
}
+ context.leafOperatorToFollowingWork.remove(parent);
context.leafOperatorToFollowingWork.put(parent, work);
root.removeParent(parent);
}
@@ -326,7 +327,7 @@ public class GenTezWork implements NodeP
MergeJoinWork mergeJoinWork = (MergeJoinWork) followingWork;
CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator();
work.setTag(mergeJoinOp.getTagForOperator(operator));
- mergeJoinWork.addMergedWork(null, work);
+ mergeJoinWork.addMergedWork(null, work, context.leafOperatorToFollowingWork);
tezWork.setVertexType(mergeJoinWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
for (BaseWork parentWork : tezWork.getParents(work)) {
TezEdgeProperty edgeProp = tezWork.getEdgeProperty(parentWork, work);
@@ -399,7 +400,7 @@ public class GenTezWork implements NodeP
return null;
}
- private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs)
+ private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs)
throws SemanticException {
int index = 0;
for (BaseWork baseWork : tezWork.getChildren(unionWork)) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Apr 14 23:36:02 2015
@@ -115,6 +115,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.Optimizer;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
@@ -1577,13 +1578,16 @@ public class SemanticAnalyzer extends Ba
// Disallow INSERT INTO on bucketized tables
boolean isAcid = isAcidTable(tab);
- if (qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()) &&
+ boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName());
+ if (isTableWrittenTo &&
tab.getNumBuckets() > 0 && !isAcid) {
throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE.
getMsg("Table: " + tab_name));
}
// Disallow update and delete on non-acid tables
- if ((updating() || deleting()) && !isAcid) {
+ if ((updating() || deleting()) && !isAcid && isTableWrittenTo) {
+ //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable)
+ //so only assert this if we are actually writing to this table
// isAcidTable above also checks for whether we are using an acid compliant
// transaction manager. But that has already been caught in
// UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
@@ -2992,7 +2996,8 @@ public class SemanticAnalyzer extends Ba
if (ensureUniqueCols) {
if (!output.putWithCheck(tmp[0], tmp[1], null, oColInfo)) {
throw new CalciteSemanticException("Cannot add column to RR: " + tmp[0] + "." + tmp[1]
- + " => " + oColInfo + " due to duplication, see previous warnings");
+ + " => " + oColInfo + " due to duplication, see previous warnings",
+ UnsupportedFeature.Duplicates_in_RR);
}
} else {
output.put(tmp[0], tmp[1], oColInfo);
@@ -11016,9 +11021,10 @@ public class SemanticAnalyzer extends Ba
// Process the position alias in GROUPBY and ORDERBY
private void processPositionAlias(ASTNode ast) throws SemanticException {
+ boolean isByPos = false;
if (HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS) == false) {
- return;
+ HiveConf.ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS) == true) {
+ isByPos = true;
}
if (ast.getChildCount() == 0) {
@@ -11052,15 +11058,20 @@ public class SemanticAnalyzer extends Ba
for (int child_pos = 0; child_pos < groupbyNode.getChildCount(); ++child_pos) {
ASTNode node = (ASTNode) groupbyNode.getChild(child_pos);
if (node.getToken().getType() == HiveParser.Number) {
- int pos = Integer.parseInt(node.getText());
- if (pos > 0 && pos <= selectExpCnt) {
- groupbyNode.setChild(child_pos,
- selectNode.getChild(pos - 1).getChild(0));
+ if (isByPos) {
+ int pos = Integer.parseInt(node.getText());
+ if (pos > 0 && pos <= selectExpCnt) {
+ groupbyNode.setChild(child_pos,
+ selectNode.getChild(pos - 1).getChild(0));
+ } else {
+ throw new SemanticException(
+ ErrorMsg.INVALID_POSITION_ALIAS_IN_GROUPBY.getMsg(
+ "Position alias: " + pos + " does not exist\n" +
+ "The Select List is indexed from 1 to " + selectExpCnt));
+ }
} else {
- throw new SemanticException(
- ErrorMsg.INVALID_POSITION_ALIAS_IN_GROUPBY.getMsg(
- "Position alias: " + pos + " does not exist\n" +
- "The Select List is indexed from 1 to " + selectExpCnt));
+ warn("Using constant number " + node.getText() +
+ " in group by. If you try to use position alias when hive.groupby.orderby.position.alias is false, the position alias will be ignored.");
}
}
}
@@ -11079,19 +11090,24 @@ public class SemanticAnalyzer extends Ba
ASTNode colNode = (ASTNode) orderbyNode.getChild(child_pos);
ASTNode node = (ASTNode) colNode.getChild(0);
if (node.getToken().getType() == HiveParser.Number) {
- if (!isAllCol) {
- int pos = Integer.parseInt(node.getText());
- if (pos > 0 && pos <= selectExpCnt) {
- colNode.setChild(0, selectNode.getChild(pos - 1).getChild(0));
+ if( isByPos ) {
+ if (!isAllCol) {
+ int pos = Integer.parseInt(node.getText());
+ if (pos > 0 && pos <= selectExpCnt) {
+ colNode.setChild(0, selectNode.getChild(pos - 1).getChild(0));
+ } else {
+ throw new SemanticException(
+ ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg(
+ "Position alias: " + pos + " does not exist\n" +
+ "The Select List is indexed from 1 to " + selectExpCnt));
+ }
} else {
throw new SemanticException(
- ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg(
- "Position alias: " + pos + " does not exist\n" +
- "The Select List is indexed from 1 to " + selectExpCnt));
+ ErrorMsg.NO_SUPPORTED_ORDERBY_ALLCOLREF_POS.getMsg());
}
- } else {
- throw new SemanticException(
- ErrorMsg.NO_SUPPORTED_ORDERBY_ALLCOLREF_POS.getMsg());
+ } else { //if not using position alias and it is a number.
+ warn("Using constant number " + node.getText() +
+ " in order by. If you try to use position alias when hive.groupby.orderby.position.alias is false, the position alias will be ignored.");
}
}
}
@@ -12092,4 +12108,8 @@ public class SemanticAnalyzer extends Ba
queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit());
}
}
+ private void warn(String msg) {
+ SessionState.getConsole().printInfo(
+ String.format("Warning: %s", msg));
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java Tue Apr 14 23:36:02 2015
@@ -140,6 +140,7 @@ public class UpdateDeleteSemanticAnalyze
}
List<FieldSchema> partCols = mTable.getPartCols();
+ List<String> bucketingCols = mTable.getBucketCols();
rewrittenQueryStr.append("insert into table ");
rewrittenQueryStr.append(getDotName(tableName));
@@ -199,7 +200,10 @@ public class UpdateDeleteSemanticAnalyze
}
}
}
-
+ //updating bucket column should move row from one file to another - not supported
+ if(bucketingCols != null && bucketingCols.contains(columnName)) {
+ throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE,columnName);
+ }
// This means that in UPDATE T SET x = _something_
// _something_ can be whatever is supported in SELECT _something_
setCols.put(columnName, (ASTNode)assignment.getChildren().get(1));
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java Tue Apr 14 23:36:02 2015
@@ -22,6 +22,7 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
public class AbstractOperatorDesc implements OperatorDesc {
@@ -33,9 +34,9 @@ public class AbstractOperatorDesc implem
static {
PTFUtils.makeTransient(AbstractOperatorDesc.class, "opProps");
}
-
+
@Override
- @Explain(skipHeader = true, displayName = "Statistics")
+ @Explain(skipHeader = true, displayName = "Statistics", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Statistics getStatistics() {
return statistics;
}
@@ -50,14 +51,18 @@ public class AbstractOperatorDesc implem
throw new CloneNotSupportedException("clone not supported");
}
+ public boolean getVectorMode() {
+ return vectorMode;
+ }
+
public void setVectorMode(boolean vm) {
this.vectorMode = vm;
}
-
+
public OpTraits getTraits() {
return opTraits;
}
-
+
public void setTraits(OpTraits opTraits) {
this.opTraits = opTraits;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterDatabaseDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterDatabaseDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterDatabaseDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterDatabaseDesc.java Tue Apr 14 23:36:02 2015
@@ -21,11 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* AlterDatabaseDesc.
*
*/
-@Explain(displayName = "Create Database")
+@Explain(displayName = "Create Database", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class AlterDatabaseDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -68,7 +70,7 @@ public class AlterDatabaseDesc extends D
this.dbProperties = dbProps;
}
- @Explain(displayName="name")
+ @Explain(displayName="name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getDatabaseName() {
return databaseName;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java Tue Apr 14 23:36:02 2015
@@ -21,11 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* AlterIndexDesc.
*
*/
-@Explain(displayName = "Alter Index")
+@Explain(displayName = "Alter Index", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class AlterIndexDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
private String indexName;
@@ -53,7 +55,7 @@ public class AlterIndexDesc extends DDLD
/**
* @return the name of the index
*/
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getIndexName() {
return indexName;
}
@@ -69,7 +71,7 @@ public class AlterIndexDesc extends DDLD
/**
* @return the baseTable
*/
- @Explain(displayName = "new name")
+ @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getBaseTableName() {
return baseTable;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java Tue Apr 14 23:36:02 2015
@@ -32,12 +32,13 @@ import org.apache.hadoop.hive.ql.exec.Ut
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* AlterTableDesc.
*
*/
-@Explain(displayName = "Alter Table")
+@Explain(displayName = "Alter Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class AlterTableDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -261,12 +262,12 @@ public class AlterTableDesc extends DDLD
this.numberBuckets = numBuckets;
}
- @Explain(displayName = "new columns")
+ @Explain(displayName = "new columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List<String> getNewColsString() {
return Utilities.getFieldSchemaString(getNewCols());
}
- @Explain(displayName = "type")
+ @Explain(displayName = "type", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getAlterTableTypeString() {
return op.getName();
}
@@ -274,7 +275,7 @@ public class AlterTableDesc extends DDLD
/**
* @return the old name of the table
*/
- @Explain(displayName = "old name")
+ @Explain(displayName = "old name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getOldName() {
return oldName;
}
@@ -290,7 +291,7 @@ public class AlterTableDesc extends DDLD
/**
* @return the newName
*/
- @Explain(displayName = "new name")
+ @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getNewName() {
return newName;
}
@@ -368,7 +369,7 @@ public class AlterTableDesc extends DDLD
/**
* @return the input format
*/
- @Explain(displayName = "input format")
+ @Explain(displayName = "input format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getInputFormat() {
return inputFormat;
}
@@ -384,7 +385,7 @@ public class AlterTableDesc extends DDLD
/**
* @return the output format
*/
- @Explain(displayName = "output format")
+ @Explain(displayName = "output format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getOutputFormat() {
return outputFormat;
}
@@ -400,7 +401,7 @@ public class AlterTableDesc extends DDLD
/**
* @return the storage handler
*/
- @Explain(displayName = "storage handler")
+ @Explain(displayName = "storage handler", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getStorageHandler() {
return storageHandler;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java Tue Apr 14 23:36:02 2015
@@ -19,12 +19,14 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* ArchiveWork.
*
*/
-@Explain(displayName = "Map Reduce")
+@Explain(displayName = "Map Reduce", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class ArchiveWork implements Serializable {
private static final long serialVersionUID = 1L;
private ArchiveActionType type;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Tue Apr 14 23:36:02 2015
@@ -29,12 +29,14 @@ import java.util.Stack;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* BaseWork. Base class for any "work" that's being done on the cluster. Items like stats
* gathering that are commonly used regardless of the type of work live here.
*/
-@SuppressWarnings({"serial", "deprecation"})
+@SuppressWarnings({"serial"})
public abstract class BaseWork extends AbstractOperatorDesc {
// dummyOps is a reference to all the HashTableDummy operators in the
@@ -58,9 +60,9 @@ public abstract class BaseWork extends A
private String name;
// Vectorization.
- protected Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = null;
- protected Map<String, Map<String, Integer>> allColumnVectorMaps = null;
- protected boolean vectorMode = false;
+ protected Map<String, Integer> vectorColumnNameMap;
+ protected Map<Integer, String> vectorColumnTypeMap;
+ protected Map<Integer, String> vectorScratchColumnTypeMap;
public void setGatheringStats(boolean gatherStats) {
this.gatheringStats = gatherStats;
@@ -142,27 +144,34 @@ public abstract class BaseWork extends A
return returnSet;
}
- public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
- return allScratchColumnVectorTypeMaps;
+ public Map<String, Integer> getVectorColumnNameMap() {
+ return vectorColumnNameMap;
+ }
+
+ public void setVectorColumnNameMap(Map<String, Integer> vectorColumnNameMap) {
+ this.vectorColumnNameMap = vectorColumnNameMap;
+ }
+
+ public Map<Integer, String> getVectorColumnTypeMap() {
+ return vectorColumnTypeMap;
}
- public void setAllScratchColumnVectorTypeMaps(
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps) {
- this.allScratchColumnVectorTypeMaps = allScratchColumnVectorTypeMaps;
+ public void setVectorColumnTypeMap(Map<Integer, String> vectorColumnTypeMap) {
+ this.vectorColumnTypeMap = vectorColumnTypeMap;
}
- public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
- return allColumnVectorMaps;
+ public Map<Integer, String> getVectorScratchColumnTypeMap() {
+ return vectorScratchColumnTypeMap;
}
- public void setAllColumnVectorMaps(Map<String, Map<String, Integer>> allColumnVectorMaps) {
- this.allColumnVectorMaps = allColumnVectorMaps;
+ public void setVectorScratchColumnTypeMap(Map<Integer, String> vectorScratchColumnTypeMap) {
+ this.vectorScratchColumnTypeMap = vectorScratchColumnTypeMap;
}
/**
* @return the mapredLocalWork
*/
- @Explain(displayName = "Local Work")
+ @Explain(displayName = "Local Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public MapredLocalWork getMapRedLocalWork() {
return mrLocalWork;
}
@@ -175,15 +184,6 @@ public abstract class BaseWork extends A
this.mrLocalWork = mapLocalWork;
}
- @Override
- public void setVectorMode(boolean vectorMode) {
- this.vectorMode = vectorMode;
- }
-
- public boolean getVectorMode() {
- return vectorMode;
- }
-
public abstract void configureJobConf(JobConf job);
public void setTag(int tag) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java Tue Apr 14 23:36:02 2015
@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.ql.exec.BucketMatcher;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* was inner class of MapreLocalWork. context for bucket mapjoin (or smb join)
@@ -130,7 +131,7 @@ public class BucketMapJoinContext implem
this.bucketMatcherClass = bucketMatcherClass;
}
- @Explain(displayName = "Alias Bucket File Name Mapping", normalExplain = false)
+ @Explain(displayName = "Alias Bucket File Name Mapping", explainLevels = { Level.EXTENDED })
public Map<String, Map<String, List<String>>> getAliasBucketFileNameMapping() {
return aliasBucketFileNameMapping;
}
@@ -149,7 +150,7 @@ public class BucketMapJoinContext implem
}
}
- @Explain(displayName = "Alias Bucket Base File Name Mapping", normalExplain = false)
+ @Explain(displayName = "Alias Bucket Base File Name Mapping", explainLevels = { Level.EXTENDED })
public Map<String, Map<String, List<String>>> getAliasBucketBaseFileNameMapping() {
return aliasBucketBaseFileNameMapping;
}
@@ -159,7 +160,7 @@ public class BucketMapJoinContext implem
this.aliasBucketBaseFileNameMapping = aliasBucketBaseFileNameMapping;
}
- @Explain(displayName = "Alias Bucket Output File Name Mapping", normalExplain = false)
+ @Explain(displayName = "Alias Bucket Output File Name Mapping", explainLevels = { Level.EXTENDED })
public Map<String, Integer> getBucketFileNameMapping() {
return bucketFileNameMapping;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java Tue Apr 14 23:36:02 2015
@@ -17,13 +17,15 @@
*/
package org.apache.hadoop.hive.ql.plan;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* CollectDesc.
*
*/
-@Explain(displayName = "Collect")
+@Explain(displayName = "Collect", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CollectDesc extends AbstractOperatorDesc {
private static final long serialVersionUID = 1L;
Integer bufferSize;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java Tue Apr 14 23:36:02 2015
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.List;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* Contains the information needed to persist column level statistics
*/
@@ -51,7 +53,7 @@ public class ColumnStatsDesc extends DDL
this.tableName = tableName;
}
- @Explain(displayName = "Is Table Level Stats", normalExplain=false)
+ @Explain(displayName = "Is Table Level Stats", explainLevels = { Level.EXTENDED })
public boolean isTblLevel() {
return isTblLevel;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java Tue Apr 14 23:36:02 2015
@@ -21,6 +21,8 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* ColumnStatsUpdateWork implementation. ColumnStatsUpdateWork will persist the
@@ -30,7 +32,7 @@ import java.util.Map;
* PARTITION(partitionId=100) UPDATE STATISTICS for column value SET
* ('maxColLen'='4444','avgColLen'='44.4');
*/
-@Explain(displayName = "Column Stats Update Work")
+@Explain(displayName = "Column Stats Update Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class ColumnStatsUpdateWork implements Serializable {
private static final long serialVersionUID = 1L;
private ColumnStatsDesc colStats;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java Tue Apr 14 23:36:02 2015
@@ -21,12 +21,14 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* ColumnStats Work.
*
*/
-@Explain(displayName = "Column Stats Work")
+@Explain(displayName = "Column Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class ColumnStatsWork implements Serializable {
private static final long serialVersionUID = 1L;
private FetchWork fWork;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java Tue Apr 14 23:36:02 2015
@@ -19,8 +19,10 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
-@Explain(displayName = "Merge Join Operator")
+
+@Explain(displayName = "Merge Join Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CommonMergeJoinDesc extends MapJoinDesc implements Serializable {
private static final long serialVersionUID = 1L;
private int numBuckets;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java Tue Apr 14 23:36:02 2015
@@ -21,12 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* CopyWork.
*
*/
-@Explain(displayName = "Copy")
+@Explain(displayName = "Copy", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CopyWork implements Serializable {
private static final long serialVersionUID = 1L;
private Path fromPath;
@@ -46,12 +47,12 @@ public class CopyWork implements Seriali
this.setErrorOnSrcEmpty(errorOnSrcEmpty);
}
- @Explain(displayName = "source")
+ @Explain(displayName = "source", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Path getFromPath() {
return fromPath;
}
- @Explain(displayName = "destination")
+ @Explain(displayName = "destination", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Path getToPath() {
return toPath;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java Tue Apr 14 23:36:02 2015
@@ -21,11 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* CreateDatabaseDesc.
*
*/
-@Explain(displayName = "Create Database")
+@Explain(displayName = "Create Database", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateDatabaseDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -75,7 +77,7 @@ public class CreateDatabaseDesc extends
this.dbProperties = dbProps;
}
- @Explain(displayName="name")
+ @Explain(displayName="name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getName() {
return databaseName;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java Tue Apr 14 23:36:02 2015
@@ -22,12 +22,13 @@ import java.io.Serializable;
import java.util.List;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* CreateFunctionDesc.
*
*/
-@Explain(displayName = "Create Function")
+@Explain(displayName = "Create Function", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateFunctionDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -50,7 +51,7 @@ public class CreateFunctionDesc implemen
this.resources = resources;
}
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getFunctionName() {
return functionName;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java Tue Apr 14 23:36:02 2015
@@ -21,13 +21,14 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.List;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
/**
* CreateMacroDesc.
*
*/
-@Explain(displayName = "Create Macro")
+@Explain(displayName = "Create Macro", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateMacroDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -52,7 +53,7 @@ public class CreateMacroDesc implements
this.body = body;
}
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getMacroName() {
return macroName;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Tue Apr 14 23:36:02 2015
@@ -40,12 +40,14 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* CreateTableDesc.
*
*/
-@Explain(displayName = "Create Table")
+@Explain(displayName = "Create Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateTableDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
private static Log LOG = LogFactory.getLog(CreateTableDesc.class);
@@ -140,12 +142,12 @@ public class CreateTableDesc extends DDL
return copy == null ? null : new ArrayList<T>(copy);
}
- @Explain(displayName = "columns")
+ @Explain(displayName = "columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List<String> getColsString() {
return Utilities.getFieldSchemaString(getCols());
}
- @Explain(displayName = "partition columns")
+ @Explain(displayName = "partition columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List<String> getPartColsString() {
return Utilities.getFieldSchemaString(getPartCols());
}
@@ -159,7 +161,7 @@ public class CreateTableDesc extends DDL
this.ifNotExists = ifNotExists;
}
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getTableName() {
return tableName;
}
@@ -188,7 +190,7 @@ public class CreateTableDesc extends DDL
this.partCols = partCols;
}
- @Explain(displayName = "bucket columns")
+ @Explain(displayName = "bucket columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List<String> getBucketCols() {
return bucketCols;
}
@@ -197,7 +199,7 @@ public class CreateTableDesc extends DDL
this.bucketCols = bucketCols;
}
- @Explain(displayName = "# buckets")
+ @Explain(displayName = "# buckets", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Integer getNumBucketsExplain() {
if (numBuckets == -1) {
return null;
@@ -268,7 +270,7 @@ public class CreateTableDesc extends DDL
this.comment = comment;
}
- @Explain(displayName = "input format")
+ @Explain(displayName = "input format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getInputFormat() {
return inputFormat;
}
@@ -277,7 +279,7 @@ public class CreateTableDesc extends DDL
this.inputFormat = inputFormat;
}
- @Explain(displayName = "output format")
+ @Explain(displayName = "output format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getOutputFormat() {
return outputFormat;
}
@@ -286,7 +288,7 @@ public class CreateTableDesc extends DDL
this.outputFormat = outputFormat;
}
- @Explain(displayName = "storage handler")
+ @Explain(displayName = "storage handler", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getStorageHandler() {
return storageHandler;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java Tue Apr 14 23:36:02 2015
@@ -21,11 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* CreateTableLikeDesc.
*
*/
-@Explain(displayName = "Create Table")
+@Explain(displayName = "Create Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateTableLikeDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
String tableName;
@@ -71,7 +73,7 @@ public class CreateTableLikeDesc extends
this.ifNotExists = ifNotExists;
}
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getTableName() {
return tableName;
}
@@ -80,7 +82,7 @@ public class CreateTableLikeDesc extends
this.tableName = tableName;
}
- @Explain(displayName = "default input format")
+ @Explain(displayName = "default input format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getDefaultInputFormat() {
return defaultInputFormat;
}
@@ -89,7 +91,7 @@ public class CreateTableLikeDesc extends
this.defaultInputFormat = inputFormat;
}
- @Explain(displayName = "default output format")
+ @Explain(displayName = "default output format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getDefaultOutputFormat() {
return defaultOutputFormat;
}
@@ -148,7 +150,7 @@ public class CreateTableLikeDesc extends
this.defaultSerdeProps = serdeProps;
}
- @Explain(displayName = "like")
+ @Explain(displayName = "like", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getLikeTableName() {
return likeTableName;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java Tue Apr 14 23:36:02 2015
@@ -24,12 +24,14 @@ import java.util.Map;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* CreateViewDesc.
*
*/
-@Explain(displayName = "Create View")
+@Explain(displayName = "Create View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateViewDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -65,7 +67,7 @@ public class CreateViewDesc extends DDLD
this.isAlterViewAs = isAlterViewAs;
}
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getViewName() {
return viewName;
}
@@ -74,7 +76,7 @@ public class CreateViewDesc extends DDLD
this.viewName = viewName;
}
- @Explain(displayName = "original text")
+ @Explain(displayName = "original text", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getViewOriginalText() {
return originalText;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java Tue Apr 14 23:36:02 2015
@@ -23,6 +23,7 @@ import java.util.HashSet;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* DDLWork.
@@ -562,7 +563,7 @@ public class DDLWork implements Serializ
/**
* @return the createTblDesc
*/
- @Explain(displayName = "Create Table Operator")
+ @Explain(displayName = "Create Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public CreateTableDesc getCreateTblDesc() {
return createTblDesc;
}
@@ -608,7 +609,7 @@ public class DDLWork implements Serializ
/**
* @return the createTblDesc
*/
- @Explain(displayName = "Create Table Operator")
+ @Explain(displayName = "Create Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public CreateTableLikeDesc getCreateTblLikeDesc() {
return createTblLikeDesc;
}
@@ -624,7 +625,7 @@ public class DDLWork implements Serializ
/**
* @return the createTblDesc
*/
- @Explain(displayName = "Create View Operator")
+ @Explain(displayName = "Create View Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public CreateViewDesc getCreateViewDesc() {
return createVwDesc;
}
@@ -640,7 +641,7 @@ public class DDLWork implements Serializ
/**
* @return the dropTblDesc
*/
- @Explain(displayName = "Drop Table Operator")
+ @Explain(displayName = "Drop Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public DropTableDesc getDropTblDesc() {
return dropTblDesc;
}
@@ -656,7 +657,7 @@ public class DDLWork implements Serializ
/**
* @return the alterTblDesc
*/
- @Explain(displayName = "Alter Table Operator")
+ @Explain(displayName = "Alter Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public AlterTableDesc getAlterTblDesc() {
return alterTblDesc;
}
@@ -672,7 +673,7 @@ public class DDLWork implements Serializ
/**
* @return the showDatabasesDesc
*/
- @Explain(displayName = "Show Databases Operator")
+ @Explain(displayName = "Show Databases Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowDatabasesDesc getShowDatabasesDesc() {
return showDatabasesDesc;
}
@@ -688,7 +689,7 @@ public class DDLWork implements Serializ
/**
* @return the showTblsDesc
*/
- @Explain(displayName = "Show Table Operator")
+ @Explain(displayName = "Show Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowTablesDesc getShowTblsDesc() {
return showTblsDesc;
}
@@ -704,7 +705,7 @@ public class DDLWork implements Serializ
/**
* @return the showColumnsDesc
*/
- @Explain(displayName = "Show Columns Operator")
+ @Explain(displayName = "Show Columns Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowColumnsDesc getShowColumnsDesc() {
return showColumnsDesc;
}
@@ -720,7 +721,7 @@ public class DDLWork implements Serializ
/**
* @return the showFuncsDesc
*/
- @Explain(displayName = "Show Function Operator")
+ @Explain(displayName = "Show Function Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowFunctionsDesc getShowFuncsDesc() {
return showFuncsDesc;
}
@@ -728,17 +729,17 @@ public class DDLWork implements Serializ
/**
* @return the showLocksDesc
*/
- @Explain(displayName = "Show Lock Operator")
+ @Explain(displayName = "Show Lock Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowLocksDesc getShowLocksDesc() {
return showLocksDesc;
}
- @Explain(displayName = "Show Compactions Operator")
+ @Explain(displayName = "Show Compactions Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowCompactionsDesc getShowCompactionsDesc() {
return showCompactionsDesc;
}
- @Explain(displayName = "Show Transactions Operator")
+ @Explain(displayName = "Show Transactions Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowTxnsDesc getShowTxnsDesc() {
return showTxnsDesc;
}
@@ -746,7 +747,7 @@ public class DDLWork implements Serializ
/**
* @return the lockTblDesc
*/
- @Explain(displayName = "Lock Table Operator")
+ @Explain(displayName = "Lock Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public LockTableDesc getLockTblDesc() {
return lockTblDesc;
}
@@ -754,7 +755,7 @@ public class DDLWork implements Serializ
/**
* @return the unlockTblDesc
*/
- @Explain(displayName = "Unlock Table Operator")
+ @Explain(displayName = "Unlock Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public UnlockTableDesc getUnlockTblDesc() {
return unlockTblDesc;
}
@@ -762,7 +763,7 @@ public class DDLWork implements Serializ
/**
* @return the descFuncDesc
*/
- @Explain(displayName = "Show Function Operator")
+ @Explain(displayName = "Show Function Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public DescFunctionDesc getDescFunctionDesc() {
return descFunctionDesc;
}
@@ -818,7 +819,7 @@ public class DDLWork implements Serializ
/**
* @return the showPartsDesc
*/
- @Explain(displayName = "Show Partitions Operator")
+ @Explain(displayName = "Show Partitions Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowPartitionsDesc getShowPartsDesc() {
return showPartsDesc;
}
@@ -834,7 +835,7 @@ public class DDLWork implements Serializ
/**
* @return the showCreateTblDesc
*/
- @Explain(displayName = "Show Create Table Operator")
+ @Explain(displayName = "Show Create Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowCreateTableDesc getShowCreateTblDesc() {
return showCreateTblDesc;
}
@@ -850,7 +851,7 @@ public class DDLWork implements Serializ
/**
* @return the showIndexesDesc
*/
- @Explain(displayName = "Show Index Operator")
+ @Explain(displayName = "Show Index Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public ShowIndexesDesc getShowIndexesDesc() {
return showIndexesDesc;
}
@@ -862,7 +863,7 @@ public class DDLWork implements Serializ
/**
* @return the descTblDesc
*/
- @Explain(displayName = "Describe Table Operator")
+ @Explain(displayName = "Describe Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public DescTableDesc getDescTblDesc() {
return descTblDesc;
}
@@ -878,7 +879,7 @@ public class DDLWork implements Serializ
/**
* @return information about the partitions we want to add.
*/
- @Explain(displayName = "Add Partition Operator")
+ @Explain(displayName = "Add Partition Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public AddPartitionDesc getAddPartitionDesc() {
return addPartitionDesc;
}
@@ -1101,7 +1102,7 @@ public class DDLWork implements Serializ
this.alterTableAlterPartDesc = alterPartitionDesc;
}
- @Explain(displayName = "Truncate Table Operator")
+ @Explain(displayName = "Truncate Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public TruncateTableDesc getTruncateTblDesc() {
return truncateTblDesc;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DemuxDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DemuxDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DemuxDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DemuxDesc.java Tue Apr 14 23:36:02 2015
@@ -20,13 +20,15 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* Demux operator descriptor implementation.
*
*/
-@Explain(displayName = "Demux Operator")
+@Explain(displayName = "Demux Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class DemuxDesc extends AbstractOperatorDesc {
private static final long serialVersionUID = 1L;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DependencyCollectionWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DependencyCollectionWork.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DependencyCollectionWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DependencyCollectionWork.java Tue Apr 14 23:36:02 2015
@@ -19,12 +19,14 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
/**
* DependencyCollectionWork
*
*/
-@Explain(displayName = "Dependency Collection")
+@Explain(displayName = "Dependency Collection", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class DependencyCollectionWork implements Serializable {
private static final long serialVersionUID = 1L;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java Tue Apr 14 23:36:02 2015
@@ -21,12 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* DescDatabaseDesc.
*
*/
-@Explain(displayName = "Describe Database")
+@Explain(displayName = "Describe Database", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class DescDatabaseDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
@@ -76,7 +77,7 @@ public class DescDatabaseDesc extends DD
/**
* @return the tableName
*/
- @Explain(displayName = "database")
+ @Explain(displayName = "database", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getDatabaseName() {
return dbName;
}
@@ -92,7 +93,7 @@ public class DescDatabaseDesc extends DD
/**
* @return the resFile
*/
- @Explain(displayName = "result file", normalExplain = false)
+ @Explain(displayName = "result file", explainLevels = { Level.EXTENDED })
public String getResFile() {
return resFile;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java?rev=1673583&r1=1673582&r2=1673583&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java Tue Apr 14 23:36:02 2015
@@ -21,12 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* DescFunctionDesc.
*
*/
-@Explain(displayName = "Describe Function")
+@Explain(displayName = "Describe Function", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class DescFunctionDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
String name;
@@ -82,7 +83,7 @@ public class DescFunctionDesc extends DD
/**
* @return the name
*/
- @Explain(displayName = "name")
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getName() {
return name;
}
@@ -98,7 +99,7 @@ public class DescFunctionDesc extends DD
/**
* @return the resFile
*/
- @Explain(displayName = "result file", normalExplain = false)
+ @Explain(displayName = "result file", explainLevels = { Level.EXTENDED })
public String getResFile() {
return resFile;
}