You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2013/05/07 17:00:07 UTC
svn commit: r1479930 - in /hive/branches/branch-0.11: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: omalley
Date: Tue May 7 15:00:06 2013
New Revision: 1479930
URL: http://svn.apache.org/r1479930
Log:
HIVE-4018 MapJoin failing with Distributed Cache error
(Amareshwari Sriramadasu via Namit Jain)
Added:
hive/branches/branch-0.11/data/files/dim-data.txt
- copied unchanged from r1471008, hive/trunk/data/files/dim-data.txt
hive/branches/branch-0.11/data/files/fact-data.txt
- copied unchanged from r1471008, hive/trunk/data/files/fact-data.txt
hive/branches/branch-0.11/ql/src/test/queries/clientpositive/join_star.q
- copied unchanged from r1471008, hive/trunk/ql/src/test/queries/clientpositive/join_star.q
hive/branches/branch-0.11/ql/src/test/results/clientpositive/join_star.q.out
- copied unchanged from r1471008, hive/trunk/ql/src/test/results/clientpositive/join_star.q.out
Modified:
hive/branches/branch-0.11/ (props changed)
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java
Propchange: hive/branches/branch-0.11/
------------------------------------------------------------------------------
Merged /hive/trunk:r1471008
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Tue May 7 15:00:06 2013
@@ -54,6 +54,7 @@ public class HashTableSinkOperator exten
private static final long serialVersionUID = 1L;
private static final Log LOG = LogFactory.getLog(HashTableSinkOperator.class.getName());
+ protected static MapJoinMetaData metadata = new MapJoinMetaData();
// from abstract map join operator
/**
* The expressions for join inputs's join keys.
@@ -164,6 +165,10 @@ public class HashTableSinkOperator exten
}
+ public static MapJoinMetaData getMetadata() {
+ return metadata;
+ }
+
private static final transient String[] FATAL_ERR_MSG = {
null, // counter value 0 means no error
"Mapside join exceeds available memory. "
@@ -301,8 +306,7 @@ public class HashTableSinkOperator exten
null);
keySerializer.initialize(null, keyTableDesc.getProperties());
- MapJoinMetaData.clear();
- MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
+ metadata.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, false, hconf));
}
@@ -349,7 +353,8 @@ public class HashTableSinkOperator exten
// Construct externalizable objects for key and value
if (needNewKey) {
- MapJoinObjectValue valueObj = new MapJoinObjectValue(metadataValueTag[tag], res);
+ MapJoinObjectValue valueObj = new MapJoinObjectValue(
+ metadataValueTag[tag], res);
rowNumber++;
if (rowNumber > hashTableScale && rowNumber % hashTableScale == 0) {
@@ -391,7 +396,7 @@ public class HashTableSinkOperator exten
.getStandardStructObjectInspector(newNames, newFields);
int alias = Integer.valueOf(metadataValueTag[tag]);
- MapJoinMetaData.put(alias, new HashTableSinkObjectCtx(
+ metadata.put(Integer.valueOf(metadataValueTag[tag]), new HashTableSinkObjectCtx(
standardOI, valueSerDe, valueTableDesc, hasFilter(alias), hconf));
}
@@ -435,7 +440,7 @@ public class HashTableSinkOperator exten
super.closeOp(abort);
} catch (Exception e) {
- LOG.error("Generate Hashtable error");
+ LOG.error("Generate Hashtable error", e);
e.printStackTrace();
}
}
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java Tue May 7 15:00:06 2013
@@ -24,20 +24,21 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
public class MapJoinMetaData {
- static transient Map<Integer, HashTableSinkObjectCtx> mapMetadata = new HashMap<Integer, HashTableSinkObjectCtx>();
+ transient Map<Integer, HashTableSinkObjectCtx> mapMetadata =
+ new HashMap<Integer, HashTableSinkObjectCtx>();
static ArrayList<Object> list = new ArrayList<Object>();
public MapJoinMetaData(){
}
- public static void put(Integer key, HashTableSinkObjectCtx value){
+ public void put(Integer key, HashTableSinkObjectCtx value){
mapMetadata.put(key, value);
}
- public static HashTableSinkObjectCtx get(Integer key){
+ public HashTableSinkObjectCtx get(Integer key){
return mapMetadata.get(key);
}
- public static void clear(){
+ public void clear(){
mapMetadata.clear();
}
@@ -45,5 +46,4 @@ public class MapJoinMetaData {
list.clear();
return list;
}
-
}
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Tue May 7 15:00:06 2013
@@ -54,6 +54,11 @@ public class MapJoinOperator extends Abs
protected transient HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue>[] mapJoinTables;
+ protected static MapJoinMetaData metadata = new MapJoinMetaData();
+ public static MapJoinMetaData getMetadata() {
+ return metadata;
+ }
+
private static final transient String[] FATAL_ERR_MSG = {
null, // counter value 0 means no error
"Mapside join exceeds available memory. "
@@ -117,7 +122,7 @@ public class MapJoinOperator extends Abs
SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
null);
keySerializer.initialize(null, keyTableDesc.getProperties());
- MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
+ metadata.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, false, hconf));
@@ -136,7 +141,7 @@ public class MapJoinOperator extends Abs
valueSerDe.initialize(null, valueTableDesc.getProperties());
ObjectInspector inspector = valueSerDe.getObjectInspector();
- MapJoinMetaData.put(Integer.valueOf(pos), new HashTableSinkObjectCtx(ObjectInspectorUtils
+ metadata.put(Integer.valueOf(pos), new HashTableSinkObjectCtx(ObjectInspectorUtils
.getStandardObjectInspector(inspector, ObjectInspectorCopyOption.WRITABLE),
valueSerDe, valueTableDesc, hasFilter(pos), hconf));
}
@@ -189,8 +194,8 @@ public class MapJoinOperator extends Abs
hashtable.initilizePersistentHash(path.toUri().getPath());
}
} catch (Exception e) {
- LOG.error("Load Distributed Cache Error");
- throw new HiveException(e.getMessage());
+ LOG.error("Load Distributed Cache Error", e);
+ throw new HiveException(e);
}
}
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java Tue May 7 15:00:06 2013
@@ -23,8 +23,10 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
-import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
+import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -93,7 +95,7 @@ public class MapJoinDoubleKeys extends A
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
try {
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = MapJoinOperator.getMetadata().get(Integer.valueOf(metadataTag));
Writable val = ctx.getSerDe().getSerializedClass().newInstance();
val.readFields(in);
@@ -124,7 +126,8 @@ public class MapJoinDoubleKeys extends A
try {
// out.writeInt(metadataTag);
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = HashTableSinkOperator.getMetadata().get(
+ Integer.valueOf(metadataTag));
ArrayList<Object> list = MapJoinMetaData.getList();
list.add(obj1);
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java Tue May 7 15:00:06 2013
@@ -23,8 +23,9 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
-import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -95,7 +96,7 @@ public class MapJoinObjectKey extends A
ClassNotFoundException {
try {
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(
+ HashTableSinkObjectCtx ctx = MapJoinOperator.getMetadata().get(
Integer.valueOf(metadataTag));
Writable val = ctx.getSerDe().getSerializedClass().newInstance();
@@ -119,7 +120,7 @@ public class MapJoinObjectKey extends A
public void writeExternal(ObjectOutput out) throws IOException {
try {
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(
+ HashTableSinkObjectCtx ctx = HashTableSinkOperator.getMetadata().get(
Integer.valueOf(metadataTag));
// Different processing for key and value
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java Tue May 7 15:00:06 2013
@@ -24,8 +24,10 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
-import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
+import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -89,7 +91,8 @@ public class MapJoinObjectValue implemen
metadataTag = in.readInt();
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = MapJoinOperator.getMetadata().get(
+ Integer.valueOf(metadataTag));
int sz = in.readInt();
MapJoinRowContainer<Object[]> res = new MapJoinRowContainer<Object[]>();
if (sz > 0) {
@@ -132,7 +135,8 @@ public class MapJoinObjectValue implemen
out.writeInt(metadataTag);
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = HashTableSinkOperator.getMetadata().get(
+ Integer.valueOf(metadataTag));
// Different processing for key and value
MapJoinRowContainer<Object[]> v = obj;
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java?rev=1479930&r1=1479929&r2=1479930&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java Tue May 7 15:00:06 2013
@@ -23,8 +23,10 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
-import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
+import org.apache.hadoop.hive.ql.exec.MapJoinMetaData;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -74,10 +76,12 @@ public class MapJoinSingleKey extends Ab
}
@Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ public void readExternal(ObjectInput in)
+ throws IOException, ClassNotFoundException {
try {
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = MapJoinOperator.getMetadata().get(
+ Integer.valueOf(metadataTag));
Writable val = ctx.getSerDe().getSerializedClass().newInstance();
val.readFields(in);
@@ -106,7 +110,8 @@ public class MapJoinSingleKey extends Ab
try {
// out.writeInt(metadataTag);
// get the tableDesc from the map stored in the mapjoin operator
- HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
+ HashTableSinkObjectCtx ctx = HashTableSinkOperator.getMetadata().get(
+ Integer.valueOf(metadataTag));
ArrayList<Object> list = MapJoinMetaData.getList();
list.add(obj);