You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/11/19 03:37:02 UTC
hive git commit: HIVE-9139: Clean up
GenSparkProcContext.clonedReduceSinks and related code [Spark Branch] (Chao
Sun, reviewed by Xuefu Zhang)
Repository: hive
Updated Branches:
refs/heads/branch-1 fa6f5cf67 -> c5b6b8df2
HIVE-9139: Clean up GenSparkProcContext.clonedReduceSinks and related code [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5b6b8df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5b6b8df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5b6b8df
Branch: refs/heads/branch-1
Commit: c5b6b8df29bbd115f6640ccb1b3fedd665527270
Parents: fa6f5cf
Author: Chao Sun <su...@apache.org>
Authored: Mon Aug 17 10:40:53 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Wed Nov 18 18:36:45 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/spark/GenSparkProcContext.java | 2 --
.../org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java | 7 -------
2 files changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c5b6b8df/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
index 0a0c791..62237e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
@@ -129,7 +129,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
public final Map<Operator<?>, BaseWork> unionWorkMap;
public final List<UnionOperator> currentUnionOperators;
public final Set<BaseWork> workWithUnionOperators;
- public final Set<ReduceSinkOperator> clonedReduceSinks;
public final Set<FileSinkOperator> fileSinkSet;
public final Map<FileSinkOperator, List<FileSinkOperator>> fileSinkMap;
@@ -180,7 +179,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
this.unionWorkMap = new LinkedHashMap<Operator<?>, BaseWork>();
this.currentUnionOperators = new LinkedList<UnionOperator>();
this.workWithUnionOperators = new LinkedHashSet<BaseWork>();
- this.clonedReduceSinks = new LinkedHashSet<ReduceSinkOperator>();
this.fileSinkSet = new LinkedHashSet<FileSinkOperator>();
this.fileSinkMap = new LinkedHashMap<FileSinkOperator, List<FileSinkOperator>>();
this.pruningSinkSet = new LinkedHashSet<Operator<?>>();
http://git-wip-us.apache.org/repos/asf/hive/blob/c5b6b8df/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
index 3dd6d92..2d5bb62 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
@@ -94,12 +94,6 @@ public class GenSparkWork implements NodeProcessor {
LOG.debug("Root operator: " + root);
LOG.debug("Leaf operator: " + operator);
- if (context.clonedReduceSinks.contains(operator)) {
- // if we're visiting a terminal we've created ourselves,
- // just skip and keep going
- return null;
- }
-
SparkWork sparkWork = context.currentTask.getWork();
SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class);
@@ -192,7 +186,6 @@ public class GenSparkWork implements NodeProcessor {
// we've already set this one up. Need to clone for the next work.
r = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(
(ReduceSinkDesc)r.getConf().clone(), r.getParentOperators());
- context.clonedReduceSinks.add(r);
}
r.getConf().setOutputName(work.getName());
}