You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/10/21 07:02:02 UTC
svn commit: r827882 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Author: namit
Date: Wed Oct 21 05:02:02 2009
New Revision: 827882
URL: http://svn.apache.org/viewvc?rev=827882&view=rev
Log:
Better error messages for debugging serde problem at reducer input
(Zheng Shao via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=827882&r1=827881&r2=827882&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Oct 21 05:02:02 2009
@@ -203,6 +203,9 @@
HIVE-890. Fix cli.sh for detecting Hadoop versions.
(Paul Huff via zshao)
+ HIVE-885. Better error messages for debugging serde problem at reducer
+ input (Zheng Shao via namit)
+
Release 0.4.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java?rev=827882&r1=827881&r2=827882&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java Wed Oct 21 05:02:02 2009
@@ -75,6 +75,9 @@
fieldNames = fieldNameArray.toArray(new String [0]);
}
+ tableDesc keyTableDesc;
+ tableDesc[] valueTableDesc;
+
public void configure(JobConf job) {
ObjectInspector[] rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE];
ObjectInspector[] valueObjectInspector = new ObjectInspector[Byte.MAX_VALUE];
@@ -98,15 +101,16 @@
reducer.setParentOperators(null); // clear out any parents as reducer is the root
isTagged = gWork.getNeedsTagging();
try {
- tableDesc keyTableDesc = gWork.getKeyDesc();
+ keyTableDesc = gWork.getKeyDesc();
inputKeyDeserializer = (SerDe)ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), null);
inputKeyDeserializer.initialize(null, keyTableDesc.getProperties());
keyObjectInspector = inputKeyDeserializer.getObjectInspector();
+ valueTableDesc = new tableDesc[gWork.getTagToValueDesc().size()];
for(int tag=0; tag<gWork.getTagToValueDesc().size(); tag++) {
// We should initialize the SerDe with the TypeInfo when available.
- tableDesc valueTableDesc = gWork.getTagToValueDesc().get(tag);
- inputValueDeserializer[tag] = (SerDe)ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
- inputValueDeserializer[tag].initialize(null, valueTableDesc.getProperties());
+ valueTableDesc[tag] = gWork.getTagToValueDesc().get(tag);
+ inputValueDeserializer[tag] = (SerDe)ReflectionUtils.newInstance(valueTableDesc[tag].getDeserializerClass(), null);
+ inputValueDeserializer[tag].initialize(null, valueTableDesc[tag].getProperties());
valueObjectInspector[tag] = inputValueDeserializer[tag].getObjectInspector();
ArrayList<ObjectInspector> ois = new ArrayList<ObjectInspector>();
@@ -180,16 +184,23 @@
try {
keyObject = inputKeyDeserializer.deserialize(keyWritable);
} catch (Exception e) {
- throw new HiveException(e);
+ throw new HiveException("Unable to deserialize reduce input key from " +
+ Utilities.formatBinaryString(keyWritable.get(), 0, keyWritable.getSize())
+ + " with properties " + keyTableDesc.getProperties(),
+ e);
}
// System.err.print(keyObject.toString());
while (values.hasNext()) {
- Writable valueWritable = (Writable) values.next();
+ BytesWritable valueWritable = (BytesWritable) values.next();
//System.err.print(who.getHo().toString());
try {
valueObject[tag.get()] = inputValueDeserializer[tag.get()].deserialize(valueWritable);
} catch (SerDeException e) {
- throw new HiveException(e);
+ throw new HiveException("Unable to deserialize reduce input value (tag=" + tag.get()
+ + ") from " +
+ Utilities.formatBinaryString(valueWritable.getBytes(), 0, valueWritable.getLength())
+ + " with properties " + valueTableDesc[tag.get()].getProperties(),
+ e);
}
row.clear();
row.add(keyObject);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=827882&r1=827881&r2=827882&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Oct 21 05:02:02 2009
@@ -777,4 +777,12 @@
curThread.setContextClassLoader(loader);
}
+ public static String formatBinaryString(byte[] array, int start, int length) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = start; i < start + length; i++) {
+ sb.append("x");
+ sb.append(array[i] < 0 ? array[i] + 256 : array[i] + 0);
+ }
+ return sb.toString();
+ }
}