You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/19 21:57:05 UTC
svn commit: r1504983 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec: JoinOperator.java
persistence/AbstractRowContainer.java persistence/MapJoinObjectValue.java
persistence/MapJoinRowContainer.java persistence/RowContainer.java
Author: hashutosh
Date: Fri Jul 19 19:57:04 2013
New Revision: 1504983
URL: http://svn.apache.org/r1504983
Log:
HIVE-4730 : Join on more than 2^31 records on single reducer failed (wrong results) (Navis via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Fri Jul 19 19:57:04 2013
@@ -87,7 +87,7 @@ public class JoinOperator extends Common
}
// number of rows for the key in the given table
- int sz = storage[alias].size();
+ long sz = storage[alias].size();
StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[tag];
StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY
.toString());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java Fri Jul 19 19:57:04 2013
@@ -38,7 +38,7 @@ public abstract class AbstractRowContain
* @return number of elements in the RowContainer
*/
- public abstract int size();
+ public abstract long size();
/**
* Remove all elements in the RowContainer.
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java Fri Jul 19 19:57:04 2013
@@ -140,7 +140,7 @@ public class MapJoinObjectValue implemen
// Different processing for key and value
MapJoinRowContainer<Object[]> v = obj;
- out.writeInt(v.size());
+ out.writeInt((int)v.size());
if (v.size() > 0) {
Object[] row = v.first();
out.writeInt(row.length);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java Fri Jul 19 19:57:04 2013
@@ -65,7 +65,7 @@ public class MapJoinRowContainer<Row> ex
* @return number of elements in the RowContainer
*/
@Override
- public int size() {
+ public long size() {
return list.size();
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Fri Jul 19 19:57:04 2013
@@ -86,7 +86,7 @@ public class RowContainer<Row extends Li
private int blockSize; // number of objects in the block before it is spilled
// to disk
private int numFlushedBlocks; // total # of blocks
- private int size; // total # of elements in the RowContainer
+ private long size; // total # of elements in the RowContainer
private File tmpFile; // temporary file holding the spilled blocks
Path tempOutPath = null;
private File parentFile;
@@ -283,7 +283,7 @@ public class RowContainer<Row extends Li
}
}
- ArrayList<Object> row = new ArrayList<Object>(2);
+ private final ArrayList<Object> row = new ArrayList<Object>(2);
private void spillBlock(Row[] block, int length) throws HiveException {
try {
@@ -360,7 +360,7 @@ public class RowContainer<Row extends Li
* @return number of elements in the RowContainer
*/
@Override
- public int size() {
+ public long size() {
return size;
}