You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/06/25 01:48:07 UTC
svn commit: r788212 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/
Author: zshao
Date: Wed Jun 24 23:48:06 2009
New Revision: 788212
URL: http://svn.apache.org/viewvc?rev=788212&view=rev
Log:
HIVE-575. Fix Map join out-of-memory problem. (Namit Jain via zshao)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObjectKey.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=788212&r1=788211&r2=788212&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Jun 24 23:48:06 2009
@@ -265,6 +265,8 @@
HIVE-573. Fix TestHiveServer. (Raghotham Murthy via zshao)
+ HIVE-575. Fix Map join out-of-memory problem. (Namit Jain via zshao)
+
Release 0.3.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObjectKey.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObjectKey.java?rev=788212&r1=788211&r2=788212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObjectKey.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObjectKey.java Wed Jun 24 23:48:06 2009
@@ -20,6 +20,7 @@
import java.io.Externalizable;
import java.io.IOException;
+import java.lang.Exception;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
@@ -70,7 +71,7 @@
}
public int hashCode() {
- return (obj == null) ? 0 : obj.hashCode();
+ return (obj == null) ? metadataTag : obj.hashCode();
}
@Override
@@ -81,6 +82,8 @@
// get the tableDesc from the map stored in the mapjoin operator
MapJoinObjectCtx ctx = MapJoinOperator.getMapMetadata().get(Integer.valueOf(metadataTag));
+
+ Writable val = ctx.getSerDe().getSerializedClass().newInstance();
val.readFields(in);
obj =
(ArrayList<Object>)
@@ -88,9 +91,10 @@
ctx.getSerDe().deserialize(val),
ctx.getSerDe().getObjectInspector(),
ObjectInspectorCopyOption.WRITABLE);
- } catch (SerDeException e) {
+ } catch (Exception e) {
throw new IOException(e);
}
+
}
@Override
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=788212&r1=788211&r2=788212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Wed Jun 24 23:48:06 2009
@@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
+import java.lang.Exception;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -44,6 +45,7 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.hive.ql.util.jdbm.htree.HTree;
+import org.apache.hadoop.hive.ql.util.jdbm.helper.FastIterator;
import org.apache.hadoop.hive.ql.util.jdbm.RecordManager;
import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerFactory;
import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerOptions;
@@ -244,6 +246,7 @@
// This may potentially increase the size of the hashmap on the mapper
if (res.size() > mapJoinRowsKey) {
LOG.warn("Number of values for a given key " + keyObj + " are " + res.size());
+ LOG.warn("used memory " + Runtime.getRuntime().totalMemory());
}
hashTable.put(keyObj, valueObj);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java?rev=788212&r1=788211&r2=788212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java Wed Jun 24 23:48:06 2009
@@ -190,7 +190,7 @@
{
Location retval = freeman.get( size );
if ( retval == null ) {
- retval = allocNew( size, pageman.getLast( Magic.USED_PAGE ) );
+ retval = allocNew( 2*size, pageman.getLast( Magic.USED_PAGE ) );
}
return retval;
}