You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/02/06 23:12:02 UTC
svn commit: r1443239 - in /hive/branches/ptf-windowing: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
metastore/src/test/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/pl...
Author: hashutosh
Date: Wed Feb 6 22:12:01 2013
New Revision: 1443239
URL: http://svn.apache.org/viewvc?rev=1443239&view=rev
Log:
Merging with latest trunk
Added:
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/
- copied from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
- copied unchanged from r1443238, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
hive/branches/ptf-windowing/ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
- copied unchanged from r1443238, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/hiveprofiler0.q
- copied unchanged from r1443238, hive/trunk/ql/src/test/queries/clientpositive/hiveprofiler0.q
hive/branches/ptf-windowing/ql/src/test/results/clientpositive/hiveprofiler0.q.out
- copied unchanged from r1443238, hive/trunk/ql/src/test/results/clientpositive/hiveprofiler0.q.out
Modified:
hive/branches/ptf-windowing/ (props changed)
hive/branches/ptf-windowing/build.properties
hive/branches/ptf-windowing/build.xml
hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/ptf-windowing/conf/hive-default.xml.template
hive/branches/ptf-windowing/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/mapjoin1.q
hive/branches/ptf-windowing/ql/src/test/results/clientpositive/mapjoin1.q.out
Propchange: hive/branches/ptf-windowing/
------------------------------------------------------------------------------
svn:mergeinfo = /hive/trunk:1441969-1443238
Modified: hive/branches/ptf-windowing/build.properties
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/build.properties?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/build.properties (original)
+++ hive/branches/ptf-windowing/build.properties Wed Feb 6 22:12:01 2013
@@ -71,6 +71,13 @@ jasperc.test.jar=${hadoop.root}/lib/jett
jsp.test.jar=${hadoop.root}/lib/jetty-ext/jsp-api.jar
common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar
+# module names needed for build process
+iterate.hive.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins
+iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins
+iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,odbc,serde,service
+iterate.hive.thrift=ql,service,metastore,serde
+iterate.hive.cpp=odbc
+
#
# Test Properties
#
Modified: hive/branches/ptf-windowing/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/build.xml?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/build.xml (original)
+++ hive/branches/ptf-windowing/build.xml Wed Feb 6 22:12:01 2013
@@ -97,59 +97,24 @@
</sequential>
</macrodef>
- <macrodef name="iterate-cpp">
- <attribute name="target"/>
- <sequential>
- <subant target="@{target}">
- <property name="thrift.home" value="${thrift.home}"/>
- <property name="build.dir.hive" location="${build.dir.hive}"/>
- <!-- TODO filelist dir="." files="metastore/build.xml,serde/build.xml,service/build.xml,odbc/build.xml"/-->
- <filelist dir="." files="odbc/build.xml"/>
- </subant>
- </sequential>
- </macrodef>
-
- <macrodef name="iterate-test">
- <attribute name="target"/>
- <sequential>
- <subant target="@{target}">
- <property name="thrift.home" value="${thrift.home}"/>
- <property name="build.dir.hive" location="${build.dir.hive}"/>
- <filelist dir="." files="ql/build.xml,contrib/build.xml,hbase-handler/build.xml,hwi/build.xml,jdbc/build.xml,metastore/build.xml,odbc/build.xml,serde/build.xml,service/build.xml"/>
- </subant>
- </sequential>
- </macrodef>
-
- <macrodef name="iterate-thriftif">
- <attribute name="target"/>
- <sequential>
- <subant target="@{target}">
- <property name="thrift.home" value="${thrift.home}"/>
- <property name="build.dir.hive" location="${build.dir.hive}"/>
- <filelist dir="." files="ql/build.xml,service/build.xml,metastore/build.xml,serde/build.xml"/>
- </subant>
- </sequential>
- </macrodef>
-
- <macrodef name="iterate-all">
- <attribute name="target"/>
- <sequential>
- <subant target="@{target}">
- <property name="build.dir.hive" location="${build.dir.hive}"/>
- <property name="is-offline" value="${is-offline}"/>
- <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml,builtins/build.xml"/>
- </subant>
- </sequential>
- </macrodef>
-
<macrodef name="iterate">
<attribute name="target"/>
+ <attribute name="iterate"/>
<sequential>
- <subant target="@{target}">
- <property name="build.dir.hive" location="${build.dir.hive}"/>
- <property name="is-offline" value="${is-offline}"/>
- <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml,builtins/build.xml"/>
- </subant>
+ <if>
+ <isset property="module"/>
+ <then> <var name="target.module" value="${module}"/> </then>
+ <else> <var name="target.module" value="@{iterate}"/> </else>
+ </if>
+ <for list="${target.module}" param="module">
+ <sequential>
+ <subant buildpath="@{module}/build.xml" target="@{target}">
+ <property name="is-offline" value="${is-offline}"/>
+ <property name="thrift.home" value="${thrift.home}"/>
+ <property name="build.dir.hive" location="${build.dir.hive}"/>
+ </subant>
+ </sequential>
+ </for>
</sequential>
</macrodef>
@@ -238,12 +203,12 @@
<target name="init" depends="ivy-init-antlib,deploy-ant-tasks">
<echo message="Project: ${ant.project.name}"/>
- <iterate-all target="init"/>
+ <iterate target="init" iterate="${iterate.hive.all}"/>
</target>
<target name="test-init">
<echo message="Project: ${ant.project.name}"/>
- <iterate-all target="test-init"/>
+ <iterate target="test-init" iterate="${iterate.hive.all}"/>
</target>
<!-- target to deploy anttasks -->
@@ -272,53 +237,58 @@
You must set thrift.home to the absolute path of your
local Thrift installation.
</fail>
- <iterate-cpp target="compile-cpp"/>
+ <iterate target="compile-cpp" iterate="${iterate.hive.cpp}"/>
</target>
<target name="compile-cpp-clean"
description="Clean CPP artifacts">
<echo message="Project: ${ant.project.name}"/>
- <iterate-cpp target="clean"/>
+ <iterate target="clean" iterate="${iterate.hive.cpp}"/>
</target>
<target name="compile" depends="compile-cpp">
<echo message="Project: ${ant.project.name}"/>
- <iterate target="compile"/>
+ <iterate target="compile" iterate="${iterate.hive.modules}"/>
</target>
<target name="thriftif">
<echo message="Project: ${ant.project.name}"/>
- <iterate-thriftif target="thriftif"/>
+ <iterate target="thriftif" iterate="${iterate.hive.thrift}"/>
</target>
<target name="jar"
depends="init"
description="Build JAR artifacts">
<echo message="Project: ${ant.project.name}"/>
- <iterate target="jar"/>
+ <iterate target="jar" iterate="${iterate.hive.modules}"/>
</target>
<target name="gen-test" depends="jar" description="Generate tests">
- <iterate-test target="gen-test"/>
+ <iterate target="gen-test" iterate="${iterate.hive.tests}"/>
</target>
<target name="jar-test" depends="gen-test"
description="Build Java test artifacts">
<echo message="Project: ${ant.project.name}"/>
- <iterate-test target="compile-test"/>
+ <iterate target="compile-test" iterate="${iterate.hive.tests}"/>
</target>
<target name="test" depends="clean-test,jar-test" description="Run tests">
<echo message="Project: ${ant.project.name}"/>
- <antcall target="test-shims">
- <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
- </antcall>
- <for keepgoing="${test.continue.on.failure}" param="file">
- <path>
- <fileset dir="." includes="*/build.xml" excludes="ant/*,odbc/*,shims/*"/>
- </path>
+ <if>
+ <isset property="module"/>
+ <else>
+ <antcall target="test-shims">
+ <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
+ </antcall>
+ </else>
+ </if>
+ <condition property="target.module" value="${module}" else="${iterate.hive.test}">
+ <isset property="module"/>
+ </condition>
+ <for list="${target.module}" param="module">
<sequential>
- <ant antfile="@{file}" target="test" inheritAll="false" inheritRefs="true">
+ <ant antfile="@{module}/build.xml" target="test" inheritAll="false" inheritRefs="true">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
</ant>
@@ -351,7 +321,7 @@
<target name="clean-test"
description="Clean test results">
<echo message="Project: ${ant.project.name}"/>
- <iterate target="clean-test"/>
+ <iterate target="clean-test" iterate="${iterate.hive.modules}"/>
<delete dir="${build.dir.hive}/test"/>
<delete dir="${hive.root}/ql/TempStatsStore"/>
</target>
@@ -359,8 +329,8 @@
<target name="clean"
description="Clean build artifacts">
<echo message="Project: ${ant.project.name}"/>
- <iterate-all target="clean"/>
- <iterate-cpp target="clean"/>
+ <iterate target="clean" iterate="${iterate.hive.all}"/>
+ <iterate target="clean" iterate="${iterate.hive.cpp}"/>
<delete dir="${target.dir}"/>
<antcall target="clean-online"/>
<antcall target="clean-offline"/>
@@ -1058,7 +1028,7 @@
<mkdir dir="${mvn.license.dir}" />
<!-- call make pom on all projects that have ivy.xml -->
- <iterate-all target="make-pom" />
+ <iterate target="make-pom" iterate="${iterate.hive.all}"/>
<!-- copy the jars -->
<copy file="${build.dir.hive}/anttasks/hive-anttasks-${version}.jar"
Modified: hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Feb 6 22:12:01 2013
@@ -539,6 +539,18 @@ public class HiveConf extends Configurat
HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G
HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true),
+ //Profiler
+ HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"),
+ HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"),
+ HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring",
+ "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create database
+ // default timeout for JDBC connection
+ HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30),
+ HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max",
+ 0), // maximum # of retries to insert/select/delete the stats DB
+ HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait",
+ 3000), // # milliseconds to wait before the next retry
+
// Statistics
HIVESTATSAUTOGATHER("hive.stats.autogather", true),
HIVESTATSDBCLASS("hive.stats.dbclass",
Modified: hive/branches/ptf-windowing/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/conf/hive-default.xml.template?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/conf/hive-default.xml.template (original)
+++ hive/branches/ptf-windowing/conf/hive-default.xml.template Wed Feb 6 22:12:01 2013
@@ -1067,6 +1067,42 @@
</property>
<property>
+ <name>hive.profiler.dbclass</name>
+ <value>jdbc:derby</value>
+ <description>The default database that stores temporary hive statistics.</description>
+</property>
+
+<property>
+ <name>hive.profiler.jdbcdriver</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+ <description>The JDBC driver for the database that stores temporary hive statistics.</description>
+</property>
+
+<property>
+ <name>hive.profiler.dbconnectionstring</name>
+ <value>jdbc:derby:;databaseName=TempStatsStore;create=true</value>
+ <description>The default connection string for the database that stores temporary hive statistics.</description>
+</property>
+
+<property>
+ <name>hive.profiler.jdbc.timeout</name>
+ <value>30</value>
+ <description>Timeout value (number of seconds) used by JDBC connection and statements.</description>
+</property>
+
+<property>
+ <name>hive.profiler.retries.max</name>
+ <value>0</value>
+ <description>Maximum number of retries when profiler publisher/aggregator got an exception updating intermediate database. Default is no tries on failures.</description>
+</property>
+
+<property>
+ <name>hive.profiler.retries.wait</name>
+ <value>3000</value>
+ <description>The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues + baseWindow * (failure + 1) * (random number between [0.0,1.0]).</description>
+</property>
+
+<property>
<name>hive.stats.dbclass</name>
<value>jdbc:derby</value>
<description>The default database that stores temporary hive statistics.</description>
Modified: hive/branches/ptf-windowing/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java (original)
+++ hive/branches/ptf-windowing/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java Wed Feb 6 22:12:01 2013
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.metastore;
+import java.io.IOException;
+import java.net.ServerSocket;
+
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileSystem;
@@ -34,9 +37,10 @@ import org.apache.hadoop.hive.shims.Shim
public class TestMetaStoreAuthorization extends TestCase {
protected HiveConf conf = new HiveConf();
- private final int port = 10000;
+ private int port;
public void setup() throws Exception {
+ port = findFreePort();
System.setProperty(HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS.varname,
"true");
conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
@@ -111,4 +115,11 @@ public class TestMetaStoreAuthorization
}
}
}
+
+ private int findFreePort() throws IOException {
+ ServerSocket socket= new ServerSocket(0);
+ int port = socket.getLocalPort();
+ socket.close();
+ return port;
+ }
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Wed Feb 6 22:12:01 2013
@@ -63,7 +63,7 @@ import org.apache.hadoop.hive.ql.parse.R
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
@@ -323,8 +323,6 @@ public class MapJoinProcessor implements
List<Operator<? extends OperatorDesc>> oldReduceSinkParentOps =
new ArrayList<Operator<? extends OperatorDesc>>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
- HashMap<Byte, HashMap<String, ExprNodeDesc>> columnTransfer =
- new HashMap<Byte, HashMap<String, ExprNodeDesc>>();
// found a source which is not to be stored in memory
if (leftSrc != null) {
@@ -334,7 +332,6 @@ public class MapJoinProcessor implements
Operator<? extends OperatorDesc> grandParentOp =
parentOp.getParentOperators().get(0);
oldReduceSinkParentOps.add(parentOp);
- grandParentOp.removeChild(parentOp);
newParentOps.add(grandParentOp);
}
@@ -347,7 +344,6 @@ public class MapJoinProcessor implements
Operator<? extends OperatorDesc> grandParentOp =
parentOp.getParentOperators().get(0);
- grandParentOp.removeChild(parentOp);
oldReduceSinkParentOps.add(parentOp);
newParentOps.add(grandParentOp);
}
@@ -360,10 +356,6 @@ public class MapJoinProcessor implements
ReduceSinkDesc rsconf = oldPar.getConf();
List<ExprNodeDesc> keys = rsconf.getKeyCols();
keyExprMap.put(pos, keys);
-
- // set column transfer
- HashMap<String, ExprNodeDesc> map = (HashMap<String, ExprNodeDesc>) oldPar.getColumnExprMap();
- columnTransfer.put(pos, map);
}
// create the map-join operator
@@ -400,31 +392,20 @@ public class MapJoinProcessor implements
}
Map<Byte, List<ExprNodeDesc>> filters = desc.getFilters();
+ Map<Byte, List<ExprNodeDesc>> newFilters = new HashMap<Byte, List<ExprNodeDesc>>();
for (Map.Entry<Byte, List<ExprNodeDesc>> entry : filters.entrySet()) {
- Byte srcAlias = entry.getKey();
- List<ExprNodeDesc> columnDescList = entry.getValue();
+ byte srcTag = entry.getKey();
+ List<ExprNodeDesc> filter = entry.getValue();
- for (ExprNodeDesc nodeExpr : columnDescList) {
- ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) nodeExpr;
- for (ExprNodeDesc childDesc : funcDesc.getChildExprs()) {
- if (!(childDesc instanceof ExprNodeColumnDesc)) {
- continue;
- }
- ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) childDesc;
- // reset columns
- String column = columnDesc.getColumn();
- String newColumn = null;
- HashMap<String, ExprNodeDesc> map = columnTransfer.get(srcAlias);
- ExprNodeColumnDesc tmpDesc = (ExprNodeColumnDesc) map.get(column);
- if (tmpDesc != null) {
- newColumn = tmpDesc.getColumn();
- }
- if (newColumn == null) {
- throw new SemanticException("No Column name found in parent reduce sink op");
- }
- columnDesc.setColumn(newColumn);
- }
- }
+ Operator<?> start = oldReduceSinkParentOps.get(srcTag);
+ Operator<?> terminal = newParentOps.get(srcTag);
+ newFilters.put(srcTag, ExprNodeDescUtils.backtrack(filter, start, terminal));
+ }
+ desc.setFilters(filters = newFilters);
+
+ // remove old parents
+ for (pos = 0; pos < newParentOps.size(); pos++) {
+ newParentOps.get(pos).removeChild(oldReduceSinkParentOps.get(pos));
}
JoinCondDesc[] joinCondns = op.getConf().getConds();
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Wed Feb 6 22:12:01 2013
@@ -20,8 +20,11 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
public class ExprNodeDescUtils {
@@ -113,4 +116,72 @@ public class ExprNodeDescUtils {
}
return null;
}
+
+ /**
+ * Convert expressions in current operator to those in terminal operator, which
+ * is an ancestor of current or null (back to top operator).
+ */
+ public static ArrayList<ExprNodeDesc> backtrack(List<ExprNodeDesc> sources,
+ Operator<?> current, Operator<?> terminal) throws SemanticException {
+ ArrayList<ExprNodeDesc> result = new ArrayList<ExprNodeDesc>();
+ for (ExprNodeDesc expr : sources) {
+ result.add(backtrack(expr, current, terminal));
+ }
+ return result;
+ }
+
+ private static ExprNodeDesc backtrack(ExprNodeDesc source, Operator<?> current,
+ Operator<?> terminal) throws SemanticException {
+ if (current == null || current == terminal) {
+ return source;
+ }
+ if (source instanceof ExprNodeGenericFuncDesc) {
+ // all children expression should be resolved
+ ExprNodeGenericFuncDesc function = (ExprNodeGenericFuncDesc) source.clone();
+ function.setChildExprs(backtrack(function.getChildren(), current, terminal));
+ return function;
+ }
+ if (source instanceof ExprNodeColumnDesc) {
+ ExprNodeColumnDesc column = (ExprNodeColumnDesc) source;
+ return backtrack(column, current, terminal);
+ }
+ if (source instanceof ExprNodeFieldDesc) {
+ // field epression should be resolved
+ ExprNodeFieldDesc field = (ExprNodeFieldDesc) source.clone();
+ field.setDesc(backtrack(field.getDesc(), current, terminal));
+ return field;
+ }
+ // constant or null expr, just return
+ return source;
+ }
+
+ // Resolve column expression to input expression by using expression mapping in current operator
+ private static ExprNodeDesc backtrack(ExprNodeColumnDesc column, Operator<?> current,
+ Operator<?> terminal) throws SemanticException {
+ if (current == null || current == terminal) {
+ return column;
+ }
+ Operator<?> parent = getSingleParent(current, terminal);
+ Map<String, ExprNodeDesc> mapping = current.getColumnExprMap();
+ if (mapping == null || !mapping.containsKey(column.getColumn())) {
+ return backtrack(column, parent, terminal); // forward
+ }
+ ExprNodeDesc mapped = mapping.get(column.getColumn());
+ return backtrack(mapped, parent, terminal); // forward with resolved expr
+ }
+
+ private static Operator<?> getSingleParent(Operator<?> current, Operator<?> terminal)
+ throws SemanticException {
+ List<Operator<?>> parents = current.getParentOperators();
+ if (parents == null || parents.isEmpty()) {
+ if (terminal != null) {
+ throw new SemanticException("Failed to meet terminal operator");
+ }
+ return null;
+ }
+ if (current.getParentOperators().size() > 1) {
+ throw new SemanticException("Met multiple parent operators");
+ }
+ return parents.get(0);
+ }
}
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/mapjoin1.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/mapjoin1.q?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/mapjoin1.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/mapjoin1.q Wed Feb 6 22:12:01 2013
@@ -1,5 +1,40 @@
set hive.mapjoin.cache.numrows=100;
SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
- FROM srcpart a
- JOIN src b ON a.key = b.key where a.ds is not null;
+ FROM srcpart a
+ JOIN src b ON a.key = b.key where a.ds is not null;
+
+set hive.outerjoin.supports.filters=true;
+
+-- const filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10;
+
+-- func filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10;
+
+-- field filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10;
+
+set hive.outerjoin.supports.filters=false;
+
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10;
+
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10;
+
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10;
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10;
Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/mapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/mapjoin1.q.out?rev=1443239&r1=1443238&r2=1443239&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/mapjoin1.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/mapjoin1.q.out Wed Feb 6 22:12:01 2013
@@ -1,6 +1,6 @@
PREHOOK: query: SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
- FROM srcpart a
- JOIN src b ON a.key = b.key where a.ds is not null
+ FROM srcpart a
+ JOIN src b ON a.key = b.key where a.ds is not null
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Input: default@srcpart
@@ -10,8 +10,8 @@ PREHOOK: Input: default@srcpart@ds=2008-
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
- FROM srcpart a
- JOIN src b ON a.key = b.key where a.ds is not null
+ FROM srcpart a
+ JOIN src b ON a.key = b.key where a.ds is not null
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Input: default@srcpart
@@ -21,3 +21,717 @@ POSTHOOK: Input: default@srcpart@ds=2008
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
1114788.0
+PREHOOK: query: -- const filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: -- const filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) true))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {true}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {true}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 val_238
+238 val_238 238 val_238
+86 val_86 86 val_86
+311 val_311 311 val_311
+311 val_311 311 val_311
+311 val_311 311 val_311
+27 val_27 27 val_27
+165 val_165 165 val_165
+165 val_165 165 val_165
+409 val_409 409 val_409
+PREHOOK: query: -- func filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: -- func filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) (< (* (. (TOK_TABLE_OR_COL b) key) 10) '1000')))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {((key * 10) < '1000')}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {((key * 10) < '1000')}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+NULL NULL 238 val_238
+NULL NULL 238 val_238
+86 val_86 86 val_86
+NULL NULL 311 val_311
+NULL NULL 311 val_311
+NULL NULL 311 val_311
+27 val_27 27 val_27
+NULL NULL 165 val_165
+NULL NULL 165 val_165
+NULL NULL 409 val_409
+PREHOOK: query: -- field filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: -- field filter on outer join
+EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION named_struct 'key' (TOK_TABLE_OR_COL key) 'value' (TOK_TABLE_OR_COL value)) kv)))) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) (> (. (. (TOK_TABLE_OR_COL b) kv) key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ filter predicates:
+ 0
+ 1 {(_col1.key > 200)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: named_struct('key',key,'value',value)
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ filter predicates:
+ 0
+ 1 {(_col1.key > 200)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 {"key":"238","value":"val_238"}
+238 val_238 238 {"key":"238","value":"val_238"}
+NULL NULL 86 {"key":"86","value":"val_86"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+NULL NULL 27 {"key":"27","value":"val_27"}
+NULL NULL 165 {"key":"165","value":"val_165"}
+NULL NULL 165 {"key":"165","value":"val_165"}
+409 val_409 409 {"key":"409","value":"val_409"}
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) true))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: true
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Filter Operator
+ predicate:
+ expr: true
+ type: boolean
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 val_238
+238 val_238 238 val_238
+86 val_86 86 val_86
+311 val_311 311 val_311
+311 val_311 311 val_311
+311 val_311 311 val_311
+27 val_27 27 val_27
+165 val_165 165 val_165
+165 val_165 165 val_165
+409 val_409 409 val_409
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) (< (* (. (TOK_TABLE_OR_COL b) key) 10) '1000')))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key * 10) < '1000')
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Filter Operator
+ predicate:
+ expr: ((key * 10) < '1000')
+ type: boolean
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+86 val_86 86 val_86
+27 val_27 27 val_27
+98 val_98 98 val_98
+98 val_98 98 val_98
+66 val_66 66 val_66
+37 val_37 37 val_37
+37 val_37 37 val_37
+15 val_15 15 val_15
+15 val_15 15 val_15
+82 val_82 82 val_82
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION named_struct 'key' (TOK_TABLE_OR_COL key) 'value' (TOK_TABLE_OR_COL value)) kv)))) b) (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) (> (. (. (TOK_TABLE_OR_COL b) kv) key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: named_struct('key',key,'value',value)
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate:
+ expr: (_col1.key > 200)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: struct<key:string,value:string>
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
+ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 {"key":"238","value":"val_238"}
+238 val_238 238 {"key":"238","value":"val_238"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+409 val_409 409 {"key":"409","value":"val_409"}
+409 val_409 409 {"key":"409","value":"val_409"}
+409 val_409 409 {"key":"409","value":"val_409"}
+255 val_255 255 {"key":"255","value":"val_255"}
+255 val_255 255 {"key":"255","value":"val_255"}