You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jp...@apache.org on 2015/01/19 20:42:22 UTC
svn commit: r1653088 [2/2] - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/ optimizer/
optimizer/correlation/ optimizer/physical/ optimizer/spark/ parse/ plan/
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java?rev=1653088&r1=1653087&r2=1653088&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java Mon Jan 19 19:42:21 2015
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.parse;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -174,10 +173,9 @@ public class TableAccessAnalyzer {
// Get the key column names for each side of the join,
// and check if the keys are all constants
// or columns (not expressions). If yes, proceed.
- QBJoinTree joinTree = pGraphContext.getJoinContext().get(op);
- assert(parentOps.size() == joinTree.getBaseSrc().length);
+ assert(parentOps.size() == op.getConf().getBaseSrc().length);
int pos = 0;
- for (String src : joinTree.getBaseSrc()) {
+ for (String src : op.getConf().getBaseSrc()) {
if (src != null) {
assert(parentOps.get(pos) instanceof ReduceSinkOperator);
ReduceSinkOperator reduceSinkOp = (ReduceSinkOperator) parentOps.get(pos);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1653088&r1=1653087&r2=1653088&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Mon Jan 19 19:42:21 2015
@@ -387,8 +387,7 @@ public abstract class TaskCompiler {
ParseContext clone = new ParseContext(conf,
pCtx.getQB(), pCtx.getParseTree(),
pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(),
- pCtx.getOpParseCtx(), pCtx.getJoinContext(),
- pCtx.getSmbMapJoinContext(),
+ pCtx.getOpParseCtx(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(),
pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(),
pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(),
pCtx.getListMapJoinOpsNoReducer(), pCtx.getGroupOpToInputTables(),
@@ -399,7 +398,7 @@ public abstract class TaskCompiler {
pCtx.getQueryProperties());
clone.setFetchTask(pCtx.getFetchTask());
clone.setLineageInfo(pCtx.getLineageInfo());
- clone.setMapJoinContext(pCtx.getMapJoinContext());
+ clone.setMapJoinOps(pCtx.getMapJoinOps());
return clone;
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java?rev=1653088&r1=1653087&r2=1653088&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java Mon Jan 19 19:42:21 2015
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.QBJoinTree;
/**
* Join operator Descriptor implementation.
@@ -88,12 +90,26 @@ public class JoinDesc extends AbstractOp
// used only for explain.
private transient ExprNodeDesc [][] joinKeys;
+
+ // Data structures coming originally from QBJoinTree
+ private transient String leftAlias;
+ private transient String[] leftAliases;
+ private transient String[] rightAliases;
+ private transient String[] baseSrc;
+ private transient String id;
+ private transient boolean mapSideJoin;
+ private transient List<String> mapAliases; //map-side join aliases
+ private transient Map<String, Operator<? extends OperatorDesc>> aliasToOpInfo;
+ private transient boolean leftInputJoin;
+ private transient List<String> streamAliases;
+
public JoinDesc() {
}
public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
List<String> outputColumnNames, final boolean noOuterJoin,
- final JoinCondDesc[] conds, final Map<Byte, List<ExprNodeDesc>> filters, ExprNodeDesc[][] joinKeys) {
+ final JoinCondDesc[] conds, final Map<Byte, List<ExprNodeDesc>> filters,
+ ExprNodeDesc[][] joinKeys) {
this.exprs = exprs;
this.outputColumnNames = outputColumnNames;
this.noOuterJoin = noOuterJoin;
@@ -509,4 +525,88 @@ public class JoinDesc extends AbstractOp
public void setFixedAsSorted(boolean fixedAsSorted) {
this.fixedAsSorted = fixedAsSorted;
}
+
+ public String[] getLeftAliases() {
+ return leftAliases;
+ }
+
+ public String[] getBaseSrc() {
+ return baseSrc;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public List<String> getMapAliases() {
+ return mapAliases;
+ }
+
+ public Map<String, Operator<? extends OperatorDesc>> getAliasToOpInfo() {
+ return aliasToOpInfo;
+ }
+
+ public boolean isLeftInputJoin() {
+ return leftInputJoin;
+ }
+
+ public String getLeftAlias() {
+ return leftAlias;
+ }
+
+ public void setLeftAlias(String leftAlias) {
+ this.leftAlias = leftAlias;
+ }
+
+ public String[] getRightAliases() {
+ return rightAliases;
+ }
+
+ public List<String> getStreamAliases() {
+ return streamAliases;
+ }
+
+ public boolean isMapSideJoin() {
+ return mapSideJoin;
+ }
+
+ public void setQBJoinTreeProps(JoinDesc joinDesc) {
+ leftAlias = joinDesc.leftAlias;
+ leftAliases = joinDesc.leftAliases;
+ rightAliases = joinDesc.rightAliases;
+ baseSrc = joinDesc.baseSrc;
+ id = joinDesc.id;
+ mapSideJoin = joinDesc.mapSideJoin;
+ mapAliases = joinDesc.mapAliases;
+ aliasToOpInfo = joinDesc.aliasToOpInfo;
+ leftInputJoin = joinDesc.leftInputJoin;
+ streamAliases = joinDesc.streamAliases;
+ }
+
+ public void setQBJoinTreeProps(QBJoinTree joinTree) {
+ leftAlias = joinTree.getLeftAlias();
+ leftAliases = joinTree.getLeftAliases();
+ rightAliases = joinTree.getRightAliases();
+ baseSrc = joinTree.getBaseSrc();
+ id = joinTree.getId();
+ mapSideJoin = joinTree.isMapSideJoin();
+ mapAliases = joinTree.getMapAliases();
+ aliasToOpInfo = joinTree.getAliasToOpInfo();
+ leftInputJoin = joinTree.getJoinSrc() != null;
+ streamAliases = joinTree.getStreamAliases();
+ }
+
+ public void cloneQBJoinTreeProps(JoinDesc joinDesc) {
+ leftAlias = joinDesc.leftAlias;
+ leftAliases = joinDesc.leftAliases == null ? null : joinDesc.leftAliases.clone();
+ rightAliases = joinDesc.rightAliases == null ? null : joinDesc.rightAliases.clone();
+ baseSrc = joinDesc.baseSrc == null ? null : joinDesc.baseSrc.clone();
+ id = joinDesc.id;
+ mapSideJoin = joinDesc.mapSideJoin;
+ mapAliases = joinDesc.mapAliases == null ? null : new ArrayList<String>(joinDesc.mapAliases);
+ aliasToOpInfo = new HashMap<String, Operator<? extends OperatorDesc>>(joinDesc.aliasToOpInfo);
+ leftInputJoin = joinDesc.leftInputJoin;
+ streamAliases = joinDesc.streamAliases == null ? null : new ArrayList<String>(joinDesc.streamAliases);
+ }
+
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1653088&r1=1653087&r2=1653088&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Mon Jan 19 19:42:21 2015
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import com.google.common.collect.Interner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -41,10 +40,11 @@ import org.apache.hadoop.hive.ql.exec.Op
import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.BucketCol;
import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.SortCol;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
-import org.apache.hadoop.hive.ql.parse.QBJoinTree;
import org.apache.hadoop.hive.ql.parse.SplitSample;
import org.apache.hadoop.mapred.JobConf;
+import com.google.common.collect.Interner;
+
/**
* MapWork represents all the information used to run a map task on the cluster.
* It is first used when the query planner breaks the logical plan into tasks and
@@ -105,8 +105,10 @@ public class MapWork extends BaseWork {
public static final int SAMPLING_ON_START = 2; // sampling on task running
// the following two are used for join processing
- private QBJoinTree joinTree;
private LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext> opParseCtxMap;
+ private boolean leftInputJoin;
+ private String[] baseSrc;
+ private List<String> mapAliases;
private boolean mapperCannotSpanPartns;
@@ -419,14 +421,6 @@ public class MapWork extends BaseWork {
return useOneNullRowInputFormat;
}
- public QBJoinTree getJoinTree() {
- return joinTree;
- }
-
- public void setJoinTree(QBJoinTree joinTree) {
- this.joinTree = joinTree;
- }
-
public void setMapperCannotSpanPartns(boolean mapperCannotSpanPartns) {
this.mapperCannotSpanPartns = mapperCannotSpanPartns;
}
@@ -579,4 +573,28 @@ public class MapWork extends BaseWork {
public boolean getDoSplitsGrouping() {
return this.doSplitsGrouping;
}
+
+ public boolean isLeftInputJoin() {
+ return leftInputJoin;
+ }
+
+ public void setLeftInputJoin(boolean leftInputJoin) {
+ this.leftInputJoin = leftInputJoin;
+ }
+
+ public String[] getBaseSrc() {
+ return baseSrc;
+ }
+
+ public void setBaseSrc(String[] baseSrc) {
+ this.baseSrc = baseSrc;
+ }
+
+ public List<String> getMapAliases() {
+ return mapAliases;
+ }
+
+ public void setMapAliases(List<String> mapAliases) {
+ this.mapAliases = mapAliases;
+ }
}