You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/28 05:59:52 UTC
svn commit: r1642263 [1/3] - in /hive/branches/spark: ./ data/conf/
data/conf/spark/ itests/qtest-spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/
ql/src/test/queries/clientpositive/ ql/s...
Author: brock
Date: Fri Nov 28 04:59:50 2014
New Revision: 1642263
URL: http://svn.apache.org/r1642263
Log:
HIVE-8836 - Enable automatic tests with remote spark client [Spark Branch]
Added:
hive/branches/spark/data/conf/spark/log4j.properties
hive/branches/spark/itests/qtest-spark/.gitignore
Modified:
hive/branches/spark/data/conf/hive-log4j.properties
hive/branches/spark/data/conf/spark/hive-site.xml
hive/branches/spark/itests/qtest-spark/pom.xml
hive/branches/spark/pom.xml
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
hive/branches/spark/ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q
hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_3.q
hive/branches/spark/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out
hive/branches/spark/ql/src/test/results/clientpositive/smb_mapjoin_3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join20.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join29.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join30.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join32.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/count.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ctas.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby10.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby8.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_position.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/having.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join20.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join38.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join40.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join_reorder.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/sample10.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_in.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/uniquejoin.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Modified: hive/branches/spark/data/conf/hive-log4j.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/data/conf/hive-log4j.properties?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/data/conf/hive-log4j.properties (original)
+++ hive/branches/spark/data/conf/hive-log4j.properties Fri Nov 28 04:59:50 2014
@@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.lo
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
#
Modified: hive/branches/spark/data/conf/spark/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/hive-site.xml?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/data/conf/spark/hive-site.xml (original)
+++ hive/branches/spark/data/conf/spark/hive-site.xml Fri Nov 28 04:59:50 2014
@@ -197,7 +197,17 @@
<property>
<name>spark.master</name>
- <value>local[4]</value>
+ <value>local-cluster[2,2,1024]</value>
+</property>
+
+<property>
+ <name>spark.akka.logLifecycleEvents</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>spark.root.dir</name>
+ <value>${spark.home}</value>
</property>
</configuration>
Added: hive/branches/spark/data/conf/spark/log4j.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/log4j.properties?rev=1642263&view=auto
==============================================================================
--- hive/branches/spark/data/conf/spark/log4j.properties (added)
+++ hive/branches/spark/data/conf/spark/log4j.properties Fri Nov 28 04:59:50 2014
@@ -0,0 +1,30 @@
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, DRFA
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.eclipse.jetty=WARN
+log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+
+
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+
+log4j.appender.DRFA.File=${spark.root.dir}/logs/spark.log
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
Added: hive/branches/spark/itests/qtest-spark/.gitignore
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/qtest-spark/.gitignore?rev=1642263&view=auto
==============================================================================
--- hive/branches/spark/itests/qtest-spark/.gitignore (added)
+++ hive/branches/spark/itests/qtest-spark/.gitignore Fri Nov 28 04:59:50 2014
@@ -0,0 +1 @@
+thirdparty/
Modified: hive/branches/spark/itests/qtest-spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/qtest-spark/pom.xml?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/itests/qtest-spark/pom.xml (original)
+++ hive/branches/spark/itests/qtest-spark/pom.xml Fri Nov 28 04:59:50 2014
@@ -339,6 +339,44 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
+ <id>download-spark</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo file="target/download.sh">
+ env
+ set -e
+ set -x
+ /bin/pwd
+ BASE_DIR=./target
+ HIVE_ROOT=$BASE_DIR/../../../
+ DOWNLOAD_DIR=./thirdparty
+ download() {
+ url=$1;
+ finalName=$2
+ tarName=$(basename $url)
+ rm -rf $BASE_DIR/$finalName
+ if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
+ then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url
+ fi
+ tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
+ mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
+ }
+ mkdir -p $DOWNLOAD_DIR
+ download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark"
+ cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+ </echo>
+ <exec executable="bash" dir="${basedir}" failonerror="true">
+ <arg line="target/download.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
<id>generate-tests-sources</id>
<phase>generate-test-sources</phase>
<configuration>
Modified: hive/branches/spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/pom.xml?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/pom.xml (original)
+++ hive/branches/spark/pom.xml Fri Nov 28 04:59:50 2014
@@ -171,6 +171,7 @@
<felix.version>2.4.0</felix.version>
<curator.version>2.6.0</curator.version>
<jsr305.version>3.0.0</jsr305.version>
+ <spark.home>${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark</spark.home>
</properties>
<repositories>
@@ -836,6 +837,7 @@
<log4j.debug>true</log4j.debug>
<!-- don't diry up /tmp -->
<java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
+ <spark.home>${spark.home}</spark.home>
<!-- Hadoop's minidfs class uses this -->
<test.build.data>${test.tmp.dir}</test.build.data>
<!-- required by QTestUtil -->
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java Fri Nov 28 04:59:50 2014
@@ -100,7 +100,7 @@ public class HiveSparkClientFactory {
for (Map.Entry<String, String> entry : hiveConf) {
String propertyName = entry.getKey();
if (propertyName.startsWith("spark")) {
- String value = entry.getValue();
+ String value = hiveConf.get(propertyName);
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load spark configuration from hive configuration (%s -> %s).",
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java Fri Nov 28 04:59:50 2014
@@ -67,6 +67,9 @@ public class SparkJobMonitor {
while (true) {
try {
JobExecutionStatus state = sparkJobStatus.getState();
+ if (LOG.isDebugEnabled()) {
+ console.printInfo("state = " + state);
+ }
if (state != null && state != JobExecutionStatus.UNKNOWN &&
(state != lastState || state == JobExecutionStatus.RUNNING)) {
lastState = state;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q Fri Nov 28 04:59:50 2014
@@ -28,14 +28,14 @@ FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1);
FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1);
SELECT * FROM dest_g2;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_3.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_3.q?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_3.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_3.q Fri Nov 28 04:59:50 2014
@@ -1,4 +1,4 @@
-
+-- SORT_QUERY_RESULTS
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out Fri Nov 28 04:59:50 2014
@@ -265,7 +265,7 @@ FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
@@ -273,7 +273,7 @@ FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -444,15 +444,12 @@ STAGE PLANS:
expressions: _col0 (type: string), _col2 (type: bigint), concat(_col0, _col3) (type: string), _col3 (type: double), _col4 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Filter Operator
predicate: (KEY._col0 >= 5) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -480,14 +477,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: double), _col4 (type: bigint)
+ key expressions: _col0 (type: string), _col1 (type: bigint)
+ sort order: ++
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string), _col3 (type: double), _col4 (type: bigint)
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: string), VALUE._col1 (type: double), VALUE._col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -534,7 +532,7 @@ PREHOOK: query: FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -547,7 +545,7 @@ POSTHOOK: query: FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
Modified: hive/branches/spark/ql/src/test/results/clientpositive/smb_mapjoin_3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/smb_mapjoin_3.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/smb_mapjoin_3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/smb_mapjoin_3.q.out Fri Nov 28 04:59:50 2014
@@ -1,8 +1,16 @@
-PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+
+
+create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@smb_bucket_1
-POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+
+
+create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@smb_bucket_1
@@ -214,12 +222,12 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL 4 val_4
+20 val_20 20 val_20
+23 val_23 23 val_23
NULL NULL 10 val_10
NULL NULL 17 val_17
NULL NULL 19 val_19
-20 val_20 20 val_20
-23 val_23 23 val_23
+NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -273,14 +281,14 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL 4 val_4
-NULL NULL 10 val_10
-NULL NULL 17 val_17
-NULL NULL 19 val_19
20 val_20 20 val_20
23 val_23 23 val_23
25 val_25 NULL NULL
30 val_30 NULL NULL
+NULL NULL 10 val_10
+NULL NULL 17 val_17
+NULL NULL 19 val_19
+NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -449,12 +457,12 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL 4 val_4
+20 val_20 20 val_20
+23 val_23 23 val_23
NULL NULL 10 val_10
NULL NULL 17 val_17
NULL NULL 19 val_19
-20 val_20 20 val_20
-23 val_23 23 val_23
+NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -508,11 +516,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL 4 val_4
-NULL NULL 10 val_10
-NULL NULL 17 val_17
-NULL NULL 19 val_19
20 val_20 20 val_20
23 val_23 23 val_23
25 val_25 NULL NULL
30 val_30 NULL NULL
+NULL NULL 10 val_10
+NULL NULL 17 val_17
+NULL NULL 19 val_19
+NULL NULL 4 val_4
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join20.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join20.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join20.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join20.q.out Fri Nov 28 04:59:50 2014
@@ -249,7 +249,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join29.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join29.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join29.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join29.q.out Fri Nov 28 04:59:50 2014
@@ -694,7 +694,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1327,7 +1327,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1969,7 +1969,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2606,7 +2606,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2738,7 +2738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3371,7 +3371,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -3504,7 +3504,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3654,7 +3654,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join30.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join30.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join30.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join30.q.out Fri Nov 28 04:59:50 2014
@@ -185,7 +185,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -221,7 +221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -341,7 +341,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -376,7 +376,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -503,8 +503,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -579,7 +579,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -718,8 +718,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -788,7 +788,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -924,8 +924,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -994,7 +994,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1130,8 +1130,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1199,7 +1199,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1336,8 +1336,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1405,7 +1405,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join32.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join32.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join32.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join32.q.out Fri Nov 28 04:59:50 2014
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -298,7 +298,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -444,7 +444,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out Fri Nov 28 04:59:50 2014
@@ -1,8 +1,8 @@
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -12,7 +12,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -28,7 +29,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: a
@@ -56,16 +57,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -73,11 +84,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -123,10 +134,10 @@ RUN: Stage-1:MAPRED
119 val_119
119 val_119
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -136,8 +147,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -153,7 +165,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: c
@@ -166,7 +178,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: value (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: a
@@ -210,16 +222,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -227,11 +249,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -290,8 +312,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -390,14 +412,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -406,8 +420,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -416,16 +428,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
@@ -441,14 +463,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -457,8 +471,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -467,16 +479,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out Fri Nov 28 04:59:50 2014
@@ -157,7 +157,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -300,9 +300,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1639,7 +1639,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out Fri Nov 28 04:59:50 2014
@@ -72,7 +72,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3)
Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
@@ -227,7 +227,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out Fri Nov 28 04:59:50 2014
@@ -96,7 +96,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -204,7 +204,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -312,7 +312,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -416,7 +416,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -613,7 +613,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -717,7 +717,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -821,7 +821,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1018,7 +1018,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out Fri Nov 28 04:59:50 2014
@@ -151,7 +151,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -262,7 +262,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -405,9 +405,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1700,7 +1700,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1811,7 +1811,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1954,9 +1954,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out Fri Nov 28 04:59:50 2014
@@ -212,7 +212,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -348,7 +348,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -499,7 +499,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -852,7 +852,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -965,7 +965,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out Fri Nov 28 04:59:50 2014
@@ -592,7 +592,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -696,7 +696,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/count.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/count.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/count.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/count.q.out Fri Nov 28 04:59:50 2014
@@ -1,8 +1,10 @@
-PREHOOK: query: create table abcd (a int, b int, c int, d int)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@abcd
-POSTHOOK: query: create table abcd (a int, b int, c int, d int)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@abcd
@@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 35 23 6
+10 100 45 4
+10 100 NULL 5
10 1000 50 1
100 100 10 3
-12 NULL 80 2
-10 100 NULL 5
-10 100 45 4
12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
PREHOOK: type: QUERY
POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
@@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
@@ -184,7 +186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ctas.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ctas.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ctas.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ctas.q.out Fri Nov 28 04:59:50 2014
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -320,7 +320,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -529,7 +529,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -716,7 +716,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby10.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby10.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby10.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby10.q.out Fri Nov 28 04:59:50 2014
@@ -268,10 +268,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -483,10 +483,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby8.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby8.q.out Fri Nov 28 04:59:50 2014
@@ -828,10 +828,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby9.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby9.q.out Fri Nov 28 04:59:50 2014
@@ -829,10 +829,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1619,10 +1619,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2409,8 +2409,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -3196,10 +3196,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out Fri Nov 28 04:59:50 2014
@@ -122,7 +122,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -199,8 +199,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -304,7 +304,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -404,10 +404,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out Fri Nov 28 04:59:50 2014
@@ -224,8 +224,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out Fri Nov 28 04:59:50 2014
@@ -271,7 +271,7 @@ FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
@@ -279,7 +279,7 @@ FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -299,9 +299,9 @@ STAGE PLANS:
Stage: Stage-5
Spark
Edges:
- Reducer 2 <- Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 2 <- Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 5
@@ -412,13 +412,11 @@ STAGE PLANS:
expressions: _col0 (type: string), _col2 (type: bigint), concat(_col0, _col3) (type: string), _col3 (type: double), _col4 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: double), _col4 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: bigint)
+ sort order: ++
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string), _col3 (type: double), _col4 (type: bigint)
Filter Operator
predicate: (KEY._col0 >= 5) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -443,9 +441,9 @@ STAGE PLANS:
Reducer 4
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: string), VALUE._col1 (type: double), VALUE._col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -531,7 +529,7 @@ PREHOOK: query: FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -544,7 +542,7 @@ POSTHOOK: query: FROM src
INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1)
INSERT OVERWRITE TABLE dest_g4 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1)
-INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1), substr(src.key,2,1) LIMIT 10
+INSERT OVERWRITE TABLE dest_h2 SELECT substr(src.key,1,1) as c1, count(DISTINCT substr(src.value,5)) as c2, concat(substr(src.key,1,1),sum(substr(src.value,5))) as c3, sum(substr(src.value, 5)) as c4, count(src.value) as c6 GROUP BY substr(src.key,1,1), substr(src.key,2,1) ORDER BY c1, c2 LIMIT 10
INSERT OVERWRITE TABLE dest_h3 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1), substr(src.key,2,1)
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src