You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/08/05 02:41:19 UTC
svn commit: r1615830 - in
/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql:
optimizer/spark/ parse/spark/ plan/
Author: brock
Date: Tue Aug 5 00:41:18 2014
New Revision: 1615830
URL: http://svn.apache.org/r1615830
Log:
HIVE-7561 - StarterProject: Move from assert to Guava Preconditions.* in Hive on Spark (Chao via Brock) [Spark Branch]
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java?rev=1615830&r1=1615829&r2=1615830&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java Tue Aug 5 00:41:18 2014
@@ -47,6 +47,8 @@ import org.apache.hadoop.hive.ql.plan.Sp
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import com.google.common.base.Preconditions;
+
public class SparkReduceSinkMapJoinProc implements NodeProcessor {
protected transient Log LOG = LogFactory.getLog(this.getClass().getName());
@@ -100,7 +102,10 @@ public class SparkReduceSinkMapJoinProc
if (context.unionWorkMap.containsKey(parentRS)) {
parentWork = context.unionWorkMap.get(parentRS);
} else {
- assert context.childToWorkMap.get(parentRS).size() == 1;
+ int workMapSize = context.childToWorkMap.get(parentRS).size();
+ Preconditions.checkArgument(workMapSize == 1,
+ "AssertionError: expected context.childToWorkMap.get(parentRS).size() to be 1, but was " +
+ workMapSize);
parentWork = context.childToWorkMap.get(parentRS).get(0);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java?rev=1615830&r1=1615829&r2=1615830&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java Tue Aug 5 00:41:18 2014
@@ -54,6 +54,8 @@ import org.apache.hadoop.hive.ql.plan.Sp
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.UnionWork;
+import com.google.common.base.Preconditions;
+
/**
* GenSparkUtils is a collection of shared helper methods to produce SparkWork
* Cloned from GenTezUtils.
@@ -90,7 +92,8 @@ public class GenSparkUtils {
}
public ReduceWork createReduceWork(GenSparkProcContext context, Operator<?> root, SparkWork sparkWork) {
- assert !root.getParentOperators().isEmpty();
+ Preconditions.checkArgument(!root.getParentOperators().isEmpty(),
+ "AssertionError: expected root.getParentOperators() to be non-empty");
boolean isAutoReduceParallelism =
context.conf.getBoolVar(HiveConf.ConfVars.TEZ_AUTO_REDUCER_PARALLELISM);
@@ -109,7 +112,9 @@ public class GenSparkUtils {
// to choose the number of reducers. In the join/union case they will
// all be -1. In sort/order case where it matters there will be only
// one parent.
- assert context.parentOfRoot instanceof ReduceSinkOperator;
+ Preconditions.checkArgument(context.parentOfRoot instanceof ReduceSinkOperator,
+ "AssertionError: expected context.parentOfRoot to be an instance of ReduceSinkOperator, but was " +
+ context.parentOfRoot.getClass().getName());
ReduceSinkOperator reduceSink = (ReduceSinkOperator) context.parentOfRoot;
reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers());
@@ -172,12 +177,15 @@ public class GenSparkUtils {
public MapWork createMapWork(GenSparkProcContext context, Operator<?> root,
SparkWork sparkWork, PrunedPartitionList partitions) throws SemanticException {
- assert root.getParentOperators().isEmpty();
+ Preconditions.checkArgument(root.getParentOperators().isEmpty(),
+ "AssertionError: expected root.getParentOperators() to be empty");
MapWork mapWork = new MapWork("Map "+ (++sequenceNumber));
logger.debug("Adding map work (" + mapWork.getName() + ") for " + root);
// map work starts with table scan operators
- assert root instanceof TableScanOperator;
+ Preconditions.checkArgument(root instanceof TableScanOperator,
+ "AssertionError: expected root to be an instance of TableScanOperator, but was " +
+ root.getClass().getName());
String alias = ((TableScanOperator)root).getConf().getAlias();
setupMapWork(mapWork, context, partitions, root, alias);
@@ -276,7 +284,8 @@ public class GenSparkUtils {
}
// we should have been able to reach the union from only one side.
- assert count <= 1;
+ Preconditions.checkArgument(count <= 1,
+ "AssertionError: expected count to be <= 1, but was " + count);
if (parent == null) {
// root operator is union (can happen in reducers)
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java?rev=1615830&r1=1615829&r2=1615830&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java Tue Aug 5 00:41:18 2014
@@ -45,11 +45,13 @@ import org.apache.hadoop.hive.ql.plan.Sp
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.UnionWork;
+import com.google.common.base.Preconditions;
+
/**
* GenSparkWork separates the operator tree into spark tasks.
* It is called once per leaf operator (operator that forces a new execution unit.)
* and break the operators into work and tasks along the way.
- *
+ *
* Cloned from GenTezWork.
*
* TODO: need to go thru this to make it fit completely to Spark.
@@ -72,8 +74,12 @@ public class GenSparkWork implements Nod
NodeProcessorCtx procContext, Object... nodeOutputs) throws SemanticException {
GenSparkProcContext context = (GenSparkProcContext) procContext;
- assert context != null && context.currentTask != null
- && context.currentRootOperator != null;
+ Preconditions.checkArgument(context != null,
+ "AssertionError: expected context to be not null");
+ Preconditions.checkArgument(context.currentTask != null,
+ "AssertionError: expected context.currentTask to be not null");
+ Preconditions.checkArgument(context.currentRootOperator != null,
+ "AssertionError: expected context.currentRootOperator to be not null");
// Operator is a file sink or reduce sink. Something that forces
// a new vertex.
@@ -209,7 +215,8 @@ public class GenSparkWork implements Nod
// we've seen this terminal before and have created a union work object.
// just need to add this work to it. There will be no children of this one
// since we've passed this operator before.
- assert operator.getChildOperators().isEmpty();
+ Preconditions.checkArgument(operator.getChildOperators().isEmpty(),
+ "AssertionError: expected operator.getChildOperators() to be empty");
unionWork = (UnionWork) context.unionWorkMap.get(operator);
} else {
@@ -249,8 +256,12 @@ public class GenSparkWork implements Nod
+" has common downstream work:"+followingWork);
// need to add this branch to the key + value info
- assert operator instanceof ReduceSinkOperator
- && followingWork instanceof ReduceWork;
+ Preconditions.checkArgument(operator instanceof ReduceSinkOperator,
+ "AssertionError: expected operator to be an instance of ReduceSinkOperator, but was " +
+ operator.getClass().getName());
+ Preconditions.checkArgument(followingWork instanceof ReduceWork,
+ "AssertionError: expected followingWork to be an instance of ReduceWork, but was " +
+ followingWork.getClass().getName());
ReduceSinkOperator rs = (ReduceSinkOperator) operator;
ReduceWork rWork = (ReduceWork) followingWork;
GenMapRedUtils.setKeyAndValueDesc(rWork, rs);
@@ -281,7 +292,9 @@ public class GenSparkWork implements Nod
// No children means we're at the bottom. If there are more operators to scan
// the next item will be a new root.
if (!operator.getChildOperators().isEmpty()) {
- assert operator.getChildOperators().size() == 1;
+ Preconditions.checkArgument(operator.getChildOperators().size() == 1,
+ "AssertionError: expected operator.getChildOperators().size() to be 1, but was " +
+ operator.getChildOperators().size());
context.parentOfRoot = operator;
context.currentRootOperator = operator.getChildOperators().get(0);
context.preceedingWork = work;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java?rev=1615830&r1=1615829&r2=1615830&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java Tue Aug 5 00:41:18 2014
@@ -50,11 +50,13 @@ import org.apache.hadoop.hive.ql.plan.St
import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.mapred.InputFormat;
+import com.google.common.base.Preconditions;
+
/**
* ProcessAnalyzeTable sets up work for the several variants of analyze table
* (normal, no scan, partial scan.) The plan at this point will be a single
* table scan operator.
- *
+ *
* TODO: cloned from tez ProcessAnalyzeTable. Need to make sure it fits to Spark.
*/
public class SparkProcessAnalyzeTable implements NodeProcessor {
@@ -83,9 +85,13 @@ public class SparkProcessAnalyzeTable im
.getInputFormatClass();
QB queryBlock = parseContext.getQB();
QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
-
+
if (parseInfo.isAnalyzeCommand()) {
- assert tableScan.getChildOperators() == null || tableScan.getChildOperators().size() == 0;
+ Preconditions.checkArgument(tableScan.getChildOperators() == null,
+ "AssertionError: expected tableScan.getChildOperators() to be null");
+ int childOpSize = tableScan.getChildOperators().size();
+ Preconditions.checkArgument(childOpSize == 0,
+ "AssertionError: expected tableScan.getChildOperators().size() to be 0, but was " + childOpSize);
String alias = null;
for (String a: parseContext.getTopOps().keySet()) {
@@ -93,7 +99,7 @@ public class SparkProcessAnalyzeTable im
alias = a;
}
}
- assert alias != null;
+ Preconditions.checkArgument(alias != null, "AssertionError: expected alias to be not null");
SparkWork sparkWork = context.currentTask.getWork();
boolean partialScan = parseInfo.isPartialScanAnalyzeCommand();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java?rev=1615830&r1=1615829&r2=1615830&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java Tue Aug 5 00:41:18 2014
@@ -33,6 +33,8 @@ import java.util.Set;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import com.google.common.base.Preconditions;
+
/**
* This class encapsulates all the work objects that can be executed
* in a single Spark job. Currently it's basically a tree with MapWork at the
@@ -175,8 +177,10 @@ public class SparkWork extends AbstractO
* getParents returns all the nodes with edges leading into work
*/
public List<BaseWork> getParents(BaseWork work) {
- assert invertedWorkGraph.containsKey(work)
- && invertedWorkGraph.get(work) != null;
+ Preconditions.checkArgument(invertedWorkGraph.containsKey(work),
+ "AssertionError: expected invertedWorkGraph.containsKey(work) to be true");
+ Preconditions.checkArgument(invertedWorkGraph.get(work) != null,
+ "AssertionError: expected invertedWorkGraph.get(work) to be not null");
return new LinkedList<BaseWork>(invertedWorkGraph.get(work));
}
@@ -184,8 +188,10 @@ public class SparkWork extends AbstractO
* getChildren returns all the nodes with edges leading out of work
*/
public List<BaseWork> getChildren(BaseWork work) {
- assert workGraph.containsKey(work)
- && workGraph.get(work) != null;
+ Preconditions.checkArgument(workGraph.containsKey(work),
+ "AssertionError: expected workGraph.containsKey(work) to be true");
+ Preconditions.checkArgument(workGraph.get(work) != null,
+ "AssertionError: expected workGraph.get(work) to be not null");
return new LinkedList<BaseWork>(workGraph.get(work));
}