You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/04/18 00:44:08 UTC
svn commit: r1674412 [1/3] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/or...
Author: gunther
Date: Fri Apr 17 22:44:07 2015
New Revision: 1674412
URL: http://svn.apache.org/r1674412
Log:
HIVE-10284: enable container reuse for grace hash join (Wei Zheng via Gunther Hagleitner)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_11.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_13.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_14.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_15.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_3.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_4.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_5.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_7.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_8.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_9.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/lvj_mapjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mrr.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_union.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_decimal_mapjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Apr 17 22:44:07 2015
@@ -711,8 +711,8 @@ public class HiveConf extends Configurat
HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true,
"Whether Hive should use memory-optimized hash table for MapJoin. Only works on Tez,\n" +
"because memory-optimized hashtable cannot be serialized."),
- HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", false, "Whether to use hybrid" +
- "grace hash join as the join method for mapjoin."),
+ HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", true, "Whether to use hybrid" +
+ "grace hash join as the join method for mapjoin. Tez only."),
HIVEHYBRIDGRACEHASHJOINMEMCHECKFREQ("hive.mapjoin.hybridgrace.memcheckfrequency", 1024, "For " +
"hybrid grace hash join, how often (how many rows apart) we check if memory is full. " +
"This number should be power of 2."),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Fri Apr 17 22:44:07 2015
@@ -37,18 +37,20 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionHandler;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
+import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition;
+import org.apache.hadoop.hive.ql.exec.persistence.KeyValueContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
-import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.KeyValueContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -63,9 +65,6 @@ import org.apache.hadoop.io.BytesWritabl
import org.apache.hadoop.io.Writable;
import org.apache.hive.common.util.ReflectionUtil;
-import static org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition;
-import static org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper;
-
/**
* Map side Join operator implementation.
*/
@@ -80,6 +79,7 @@ public class MapJoinOperator extends Abs
private transient ObjectCache cache;
protected HashTableLoader loader;
+ private boolean loadCalled;
protected transient MapJoinTableContainer[] mapJoinTables;
private transient MapJoinTableContainerSerDe[] mapJoinTableSerdes;
@@ -88,7 +88,6 @@ public class MapJoinOperator extends Abs
private UnwrapRowContainer[] unwrapContainer;
private transient Configuration hconf;
- private transient boolean useHybridGraceHashJoin; // whether Hybrid Grace Hash Join is enabled
private transient boolean hybridMapJoinLeftover; // whether there's spilled data to be processed
private transient MapJoinBytesTableContainer currentSmallTable; // reloaded hashmap from disk
private transient int tag; // big table alias
@@ -141,15 +140,13 @@ public class MapJoinOperator extends Abs
mapJoinTables = new MapJoinTableContainer[tagLen];
mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen];
hashTblInitedOnce = false;
- useHybridGraceHashJoin =
- HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN);
generateMapMetaData();
final ExecMapperContext mapContext = getExecContext();
final MapredContext mrContext = MapredContext.get();
- if (!conf.isBucketMapJoin() && !useHybridGraceHashJoin) {
+ if (!conf.isBucketMapJoin()) {
/*
* The issue with caching in case of bucket map join is that different tasks
* process different buckets and if the container is reused to join a different bucket,
@@ -187,8 +184,22 @@ public class MapJoinOperator extends Abs
if (os.length != 0) {
Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair =
(Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>) os[0];
- mapJoinTables = pair.getLeft();
- mapJoinTableSerdes = pair.getRight();
+
+ boolean spilled = false;
+ for (MapJoinTableContainer container : pair.getLeft()) {
+ if (container != null) {
+ spilled = spilled || container.hasSpill();
+ }
+ }
+
+ if (!loadCalled && spilled) {
+ // we can't use the cached table because it has spilled.
+ loadHashTable(getExecContext(), MapredContext.get());
+ } else {
+ // let's use the table from the cache.
+ mapJoinTables = pair.getLeft();
+ mapJoinTableSerdes = pair.getRight();
+ }
hashTblInitedOnce = true;
}
@@ -258,6 +269,8 @@ public class MapJoinOperator extends Abs
private Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> loadHashTable(
ExecMapperContext mapContext, MapredContext mrContext) throws HiveException {
+ loadCalled = true;
+
if (this.hashTblInitedOnce
&& ((mapContext == null) || (mapContext.getLocalWork() == null) || (mapContext
.getLocalWork().getInputFileChangeSensitive() == false))) {
@@ -359,8 +372,8 @@ public class MapJoinOperator extends Abs
if (!noOuterJoin) {
// For Hybrid Grace Hash Join, during the 1st round processing,
// we only keep the LEFT side if the row is not spilled
- if (!useHybridGraceHashJoin || hybridMapJoinLeftover ||
- (!hybridMapJoinLeftover && joinResult != JoinUtil.JoinResult.SPILL)) {
+ if (!conf.isHybridHashJoin() || hybridMapJoinLeftover
+ || (!hybridMapJoinLeftover && joinResult != JoinUtil.JoinResult.SPILL)) {
joinNeeded = true;
storage[pos] = dummyObjVectors[pos];
}
@@ -414,47 +427,57 @@ public class MapJoinOperator extends Abs
@Override
public void closeOp(boolean abort) throws HiveException {
- for (MapJoinTableContainer tableContainer : mapJoinTables) {
- if (tableContainer != null) {
- tableContainer.dumpMetrics();
-
- if (tableContainer instanceof HybridHashTableContainer) {
- HybridHashTableContainer hybridHtContainer = (HybridHashTableContainer) tableContainer;
- hybridHtContainer.dumpStats();
-
- HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions();
- // Clear all in memory partitions first
- for (int i = 0; i < hashPartitions.length; i++) {
- if (!hashPartitions[i].isHashMapOnDisk()) {
- hybridHtContainer.setTotalInMemRowCount(
- hybridHtContainer.getTotalInMemRowCount() -
- hashPartitions[i].getHashMapFromMemory().getNumValues());
- hashPartitions[i].getHashMapFromMemory().clear();
+
+ boolean spilled = false;
+ for (MapJoinTableContainer container: mapJoinTables) {
+ if (container != null) {
+ spilled = spilled || container.hasSpill();
+ container.dumpMetrics();
+ }
+ }
+
+ if (spilled) {
+ for (MapJoinTableContainer tableContainer : mapJoinTables) {
+ if (tableContainer != null) {
+ if (tableContainer instanceof HybridHashTableContainer) {
+ HybridHashTableContainer hybridHtContainer = (HybridHashTableContainer) tableContainer;
+ hybridHtContainer.dumpStats();
+
+ HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions();
+ // Clear all in memory partitions first
+ for (int i = 0; i < hashPartitions.length; i++) {
+ if (!hashPartitions[i].isHashMapOnDisk()) {
+ hybridHtContainer.setTotalInMemRowCount(
+ hybridHtContainer.getTotalInMemRowCount() -
+ hashPartitions[i].getHashMapFromMemory().getNumValues());
+ hashPartitions[i].getHashMapFromMemory().clear();
+ }
}
- }
- assert hybridHtContainer.getTotalInMemRowCount() == 0;
+ assert hybridHtContainer.getTotalInMemRowCount() == 0;
- for (int i = 0; i < hashPartitions.length; i++) {
- if (hashPartitions[i].isHashMapOnDisk()) {
- // Recursively process on-disk triplets (hash partition, sidefile, matchfile)
- try {
- hybridMapJoinLeftover = true;
- hashMapRowGetters[smallTable] = null;
- continueProcess(hashPartitions[i], hybridHtContainer);
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- } catch (SerDeException e) {
- e.printStackTrace();
+ for (int i = 0; i < hashPartitions.length; i++) {
+ if (hashPartitions[i].isHashMapOnDisk()) {
+ // Recursively process on-disk triplets (hash partition, sidefile, matchfile)
+ try {
+ hybridMapJoinLeftover = true;
+ hashMapRowGetters[smallTable] = null;
+ continueProcess(hashPartitions[i], hybridHtContainer);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ } catch (SerDeException e) {
+ e.printStackTrace();
+ }
}
+ hybridMapJoinLeftover = false;
+ currentSmallTable = null;
}
- hybridMapJoinLeftover = false;
- currentSmallTable = null;
}
}
}
}
+
if ((this.getExecContext() != null) && (this.getExecContext().getLocalWork() != null)
&& (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
&& mapJoinTables != null) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java Fri Apr 17 22:44:07 2015
@@ -19,7 +19,14 @@
package org.apache.hadoop.hive.ql.exec.persistence;
-import com.esotericsoftware.kryo.Kryo;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,26 +40,20 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.WriteBuffers;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
-import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct;
+import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.*;
+import com.esotericsoftware.kryo.Kryo;
/**
* Hash table container that can have many partitions -- each partition has its own hashmap,
@@ -67,16 +68,16 @@ import java.util.*;
public class HybridHashTableContainer implements MapJoinTableContainer {
private static final Log LOG = LogFactory.getLog(HybridHashTableContainer.class);
- private HashPartition[] hashPartitions; // an array of partitions holding the triplets
+ private final HashPartition[] hashPartitions; // an array of partitions holding the triplets
private int totalInMemRowCount = 0; // total number of small table rows in memory
- private long memoryThreshold; // the max memory limit allocated
- private long tableRowSize; // row size of the small table
+ private final long memoryThreshold; // the max memory limit allocated
+ private final long tableRowSize; // row size of the small table
private boolean isSpilled; // whether there's any spilled partition
private int toSpillPartitionId; // the partition into which to spill the big table row;
// This may change after every setMapJoinKey call
private int numPartitionsSpilled; // number of spilled partitions
private boolean lastPartitionInMem; // only one (last one) partition is left in memory
- private int memoryCheckFrequency; // how often (# of rows apart) to check if memory is full
+ private final int memoryCheckFrequency; // how often (# of rows apart) to check if memory is full
/** The OI used to deserialize values. We never deserialize keys. */
private LazyBinaryStructObjectInspector internalValueOi;
@@ -182,6 +183,11 @@ public class HybridHashTableContainer im
long noConditionalTaskThreshold, int memCheckFreq, long tableSize,
long keyCount, long memUsage) throws SerDeException {
+ if (wbSize > noConditionalTaskThreshold) {
+ LOG.warn("adjusting hash table write buffer size to be smaller than noconditionaltasksize");
+ wbSize = (int) noConditionalTaskThreshold;
+ }
+
int newKeyCount = HashMapWrapper.calculateTableSize(
keyCountAdj, threshold, loadFactor, keyCount);
@@ -204,6 +210,7 @@ public class HybridHashTableContainer im
if (isHashMapSpilledOnCreation(i)) {
numPartitionsSpilledOnCreation++;
numPartitionsSpilled++;
+ this.setSpill(true);
} else {
memoryAllocated += hashPartitions[i].hashMap.memorySize();
}
@@ -398,8 +405,8 @@ public class HybridHashTableContainer im
*/
private int calcNumPartitions(long dataSize, int wbSize) {
if (memoryThreshold < wbSize) {
- throw new RuntimeException("Available memory is less than hashtable writebuffer size!" +
- " Try increasing hive.auto.convert.join.noconditionaltask.size.");
+ throw new IllegalStateException("Available memory is less than hashtable writebuffer size!"
+ + " Try increasing hive.auto.convert.join.noconditionaltask.size.");
}
int lowerLimit = 2;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Fri Apr 17 22:44:07 2015
@@ -29,11 +29,11 @@ import org.apache.hadoop.hive.ql.exec.Ma
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
-import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -77,8 +77,7 @@ public class HashTableLoader implements
boolean useOptimizedTables = HiveConf.getBoolVar(
hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
- boolean useHybridGraceHashJoin = HiveConf.getBoolVar(
- hconf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN);
+ boolean useHybridGraceHashJoin = desc.isHybridHashJoin();
boolean isFirstKey = true;
// Disable hybrid grace hash join for n-way join
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Fri Apr 17 22:44:07 2015
@@ -626,6 +626,8 @@ public class ConvertJoinMapJoin implemen
MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, joinOp,
joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
joinOp.getConf().getMapAliases(), bigTablePosition, true);
+ mapJoinOp.getConf().setHybridHashJoin(HiveConf.getBoolVar(context.conf,
+ HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN));
Operator<? extends OperatorDesc> parentBigTableOp =
mapJoinOp.getParentOperators().get(bigTablePosition);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java Fri Apr 17 22:44:07 2015
@@ -70,6 +70,8 @@ public class MapJoinDesc extends JoinDes
private float hashtableMemoryUsage;
protected boolean genJoinKeys = true;
+ private boolean isHybridHashJoin;
+
public MapJoinDesc() {
bigTableBucketNumMapping = new LinkedHashMap<String, Integer>();
}
@@ -90,6 +92,8 @@ public class MapJoinDesc extends JoinDes
this.parentToInput = clone.parentToInput;
this.parentKeyCounts = clone.parentKeyCounts;
this.parentDataSizes = clone.parentDataSizes;
+ this.isBucketMapJoin = clone.isBucketMapJoin;
+ this.isHybridHashJoin = clone.isHybridHashJoin;
}
public MapJoinDesc(final Map<Byte, List<ExprNodeDesc>> keys,
@@ -322,6 +326,15 @@ public class MapJoinDesc extends JoinDes
this.isBucketMapJoin = isBucketMapJoin;
}
+ @Explain(displayName = "HybridGraceHashJoin", displayOnlyOnTrue = true)
+ public boolean isHybridHashJoin() {
+ return isHybridHashJoin;
+ }
+
+ public void setHybridHashJoin(boolean isHybridHashJoin) {
+ this.isHybridHashJoin = isHybridHashJoin;
+ }
+
public void setHashTableMemoryUsage(float hashtableMemoryUsage) {
this.hashtableMemoryUsage = hashtableMemoryUsage;
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q Fri Apr 17 22:44:07 2015
@@ -22,6 +22,7 @@ where cint not in (
set hive.auto.convert.join=true;
set hive.vectorized.execution.enabled=true;
+set hive.mapjoin.hybridgrace.hashtable=false;
explain
select store.s_city, ss_net_profit
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out Fri Apr 17 22:44:07 2015
@@ -164,6 +164,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -384,6 +385,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1199,6 +1201,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out Fri Apr 17 22:44:07 2015
@@ -144,6 +144,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -961,6 +962,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out Fri Apr 17 22:44:07 2015
@@ -144,6 +144,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -315,6 +316,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1078,6 +1080,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out Fri Apr 17 22:44:07 2015
@@ -160,6 +160,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -331,6 +332,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1094,6 +1096,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out Fri Apr 17 22:44:07 2015
@@ -125,6 +125,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -286,6 +287,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -777,6 +779,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out Fri Apr 17 22:44:07 2015
@@ -177,6 +177,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -399,6 +400,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1357,6 +1359,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out Fri Apr 17 22:44:07 2015
@@ -177,6 +177,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -399,6 +400,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1359,6 +1361,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out Fri Apr 17 22:44:07 2015
@@ -473,6 +473,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out Fri Apr 17 22:44:07 2015
@@ -457,6 +457,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out Fri Apr 17 22:44:07 2015
@@ -454,6 +454,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out Fri Apr 17 22:44:07 2015
@@ -470,6 +470,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out Fri Apr 17 22:44:07 2015
@@ -332,6 +332,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out Fri Apr 17 22:44:07 2015
@@ -407,6 +407,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out Fri Apr 17 22:44:07 2015
@@ -538,6 +538,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out Fri Apr 17 22:44:07 2015
@@ -538,6 +538,7 @@ STAGE PLANS:
0 key (type: string)
1 key (type: string)
Position of Big Table: 1
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out Fri Apr 17 22:44:07 2015
@@ -129,6 +129,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col6, _col7
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col6 (type: int), _col7 (type: string), '2010-10-15' (type: string), _col0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -288,6 +289,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col6, _col7
Position of Big Table: 1
+ BucketMapJoin: true
Select Operator
expressions: _col6 (type: int), _col7 (type: string), '2010-10-15' (type: string), _col0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Modified: hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out Fri Apr 17 22:44:07 2015
@@ -144,6 +144,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col7
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col7 (type: string)
outputColumnNames: _col0, _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out Fri Apr 17 22:44:07 2015
@@ -156,6 +156,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col7
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col7 (type: string)
outputColumnNames: _col0, _col1
@@ -416,6 +417,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col1, _col7
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
outputColumnNames: _col0, _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out Fri Apr 17 22:44:07 2015
@@ -144,6 +144,7 @@ STAGE PLANS:
1 value (type: int)
outputColumnNames: _col0, _col1, _col5, _col6
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Modified: hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out Fri Apr 17 22:44:07 2015
@@ -116,6 +116,7 @@ STAGE PLANS:
1 key (type: int)
outputColumnNames: _col0, _col1, _col5, _col6
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -372,6 +373,7 @@ STAGE PLANS:
1 key (type: int), key2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -576,6 +578,7 @@ STAGE PLANS:
1 key2 (type: int), key (type: int)
outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8
Position of Big Table: 0
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out Fri Apr 17 22:44:07 2015
@@ -134,6 +134,7 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
Position of Big Table: 0
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out Fri Apr 17 22:44:07 2015
@@ -170,6 +170,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -401,6 +402,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out Fri Apr 17 22:44:07 2015
@@ -150,6 +150,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out Fri Apr 17 22:44:07 2015
@@ -150,6 +150,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -332,6 +333,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out Fri Apr 17 22:44:07 2015
@@ -166,6 +166,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -348,6 +349,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out Fri Apr 17 22:44:07 2015
@@ -131,6 +131,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -303,6 +304,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out Fri Apr 17 22:44:07 2015
@@ -183,6 +183,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -416,6 +417,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out Fri Apr 17 22:44:07 2015
@@ -183,6 +183,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -416,6 +417,7 @@ STAGE PLANS:
1 key (type: string)
Position of Big Table: 0
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out Fri Apr 17 22:44:07 2015
@@ -148,6 +148,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col7
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col7 (type: string)
outputColumnNames: _col0, _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out Fri Apr 17 22:44:07 2015
@@ -160,6 +160,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col7
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col7 (type: string)
outputColumnNames: _col0, _col1
@@ -426,6 +427,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col7
Position of Big Table: 0
Statistics: Num rows: 1696 Data size: 18097 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
outputColumnNames: _col0, _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out Fri Apr 17 22:44:07 2015
@@ -150,6 +150,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col5, _col6
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out Fri Apr 17 22:44:07 2015
@@ -122,6 +122,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col5, _col6
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -390,6 +391,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8
Position of Big Table: 0
Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -606,6 +608,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8
Position of Big Table: 0
Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out Fri Apr 17 22:44:07 2015
@@ -140,6 +140,7 @@ STAGE PLANS:
1 key (type: int)
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -212,31 +213,27 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: bigint)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
Stage: Stage-0
Fetch Operator
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out Fri Apr 17 22:44:07 2015
@@ -56,6 +56,7 @@ STAGE PLANS:
input vertices:
1 Map 4
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
sort order: ++++
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join1.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join1.q.out Fri Apr 17 22:44:07 2015
@@ -49,6 +49,7 @@ STAGE PLANS:
input vertices:
1 Map 2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out Fri Apr 17 22:44:07 2015
@@ -2713,6 +2713,7 @@ STAGE PLANS:
1 Map 3
2 Map 4
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -2834,6 +2835,7 @@ STAGE PLANS:
0 Map 1
2 Map 4
Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -2945,6 +2947,7 @@ STAGE PLANS:
1 Map 3
2 Map 4
Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out Fri Apr 17 22:44:07 2015
@@ -76,6 +76,7 @@ STAGE PLANS:
input vertices:
1 Reducer 5
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: sum(hash(_col2,_col3))
mode: hash
@@ -209,6 +210,7 @@ STAGE PLANS:
input vertices:
1 Reducer 5
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: sum(hash(_col2,_col3))
mode: hash
@@ -353,6 +355,7 @@ STAGE PLANS:
input vertices:
0 Reducer 2
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: sum(hash(_col2,_col3))
mode: hash
@@ -507,6 +510,7 @@ STAGE PLANS:
1 Reducer 5
2 Reducer 7
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: sum(hash(_col2,_col3))
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_1.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_1.q.out Fri Apr 17 22:44:07 2015
@@ -244,6 +244,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -479,6 +480,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -784,6 +786,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out Fri Apr 17 22:44:07 2015
@@ -97,6 +97,7 @@ STAGE PLANS:
1 _col0 (type: int)
input vertices:
1 Map 5
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -124,6 +125,7 @@ STAGE PLANS:
1 _col0 (type: int)
input vertices:
1 Map 5
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -276,6 +278,7 @@ STAGE PLANS:
input vertices:
0 Reducer 2
Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_11.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_11.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_11.q.out Fri Apr 17 22:44:07 2015
@@ -239,6 +239,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -549,6 +550,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -858,6 +860,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1185,6 +1188,7 @@ STAGE PLANS:
2 Map 4
Position of Big Table: 1
Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out Fri Apr 17 22:44:07 2015
@@ -308,6 +308,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col6 (type: string), _col6 (type: string)
sort order: ++
@@ -389,6 +390,7 @@ STAGE PLANS:
0 Map 2
Position of Big Table: 1
Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -399,6 +401,7 @@ STAGE PLANS:
1 Map 5
Position of Big Table: 0
Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_13.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_13.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_13.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_13.q.out Fri Apr 17 22:44:07 2015
@@ -105,6 +105,7 @@ STAGE PLANS:
input vertices:
1 Map 2
Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -314,6 +315,7 @@ STAGE PLANS:
input vertices:
1 Map 2
Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -523,6 +525,7 @@ STAGE PLANS:
input vertices:
1 Map 2
Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_14.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_14.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_14.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_14.q.out Fri Apr 17 22:44:07 2015
@@ -68,6 +68,7 @@ STAGE PLANS:
input vertices:
1 Map 3
Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -173,6 +174,7 @@ STAGE PLANS:
input vertices:
0 Map 1
Statistics: Num rows: 207 Data size: 2080 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_15.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_15.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_15.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_15.q.out Fri Apr 17 22:44:07 2015
@@ -66,6 +66,7 @@ STAGE PLANS:
input vertices:
1 Map 3
Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -148,6 +149,7 @@ STAGE PLANS:
input vertices:
0 Map 1
Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_2.q.out?rev=1674412&r1=1674411&r2=1674412&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_2.q.out Fri Apr 17 22:44:07 2015
@@ -154,6 +154,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -461,6 +462,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
Group By Operator
aggregations: count()
mode: hash