You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2014/10/30 19:27:59 UTC
svn commit: r1635577 - in /hive/trunk/hcatalog:
core/src/main/java/org/apache/hive/hcatalog/data/
hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/
hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/
Author: daijy
Date: Thu Oct 30 18:27:59 2014
New Revision: 1635577
URL: http://svn.apache.org/r1635577
Log:
HIVE-7282: HCatLoader fail to load Orc map with null key (Daniel Dai)
Modified:
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java?rev=1635577&r1=1635576&r2=1635577&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java (original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java Thu Oct 30 18:27:59 2014
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.TreeMap;
+import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -212,7 +212,7 @@ public class HCatRecordSerDe implements
private static Map<?, ?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException {
ObjectInspector koi = moi.getMapKeyObjectInspector();
ObjectInspector voi = moi.getMapValueObjectInspector();
- Map<Object, Object> m = new TreeMap<Object, Object>();
+ Map<Object, Object> m = new HashMap<Object, Object>();
Map<?, ?> readMap = moi.getMap(f);
if (readMap == null) {
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1635577&r1=1635576&r2=1635577&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java Thu Oct 30 18:27:59 2014
@@ -480,7 +480,9 @@ class PigHCatUtil {
Map<String, Object> result = new HashMap<String, Object>();
for (Entry<?, ?> entry : map.entrySet()) {
// since map key for Pig has to be Strings
- result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+ if (entry.getKey()!=null) {
+ result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+ }
}
return result;
}
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1635577&r1=1635576&r2=1635577&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Oct 30 18:27:59 2014
@@ -18,8 +18,6 @@
*/
package org.apache.hive.hcatalog.pig;
-import com.google.common.collect.ImmutableSet;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -52,7 +50,6 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -79,11 +76,13 @@ public class TestHCatLoaderComplexSchema
add("testSyntheticComplexSchema");
add("testTupleInBagInTupleInBag");
add("testMapWithComplexData");
+ add("testMapNullKey");
}});
put(IOConstants.PARQUETFILE, new HashSet<String>() {{
add("testSyntheticComplexSchema");
add("testTupleInBagInTupleInBag");
add("testMapWithComplexData");
+ add("testMapNullKey");
}});
}};
@@ -223,6 +222,10 @@ public class TestHCatLoaderComplexSchema
private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer)
throws IOException, CommandNeedRetryException, ExecException, FrontendException {
+ verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer);
+ }
+ private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, List<Tuple> result, boolean provideSchemaToStorer)
+ throws IOException, CommandNeedRetryException, ExecException, FrontendException {
MockLoader.setData(tablename + "Input", data);
try {
createTable(tablename, tableSchema);
@@ -244,7 +247,7 @@ public class TestHCatLoaderComplexSchema
Iterator<Tuple> it = server.openIterator("X");
int i = 0;
while (it.hasNext()) {
- Tuple input = data.get(i++);
+ Tuple input = result.get(i++);
Tuple output = it.next();
compareTuples(input, output);
LOG.info("tuple : {} ", output);
@@ -354,4 +357,40 @@ public class TestHCatLoaderComplexSchema
verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true);
verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false);
}
+
+ /**
+ * artificially complex nested schema to test nested schema conversion
+ * @throws Exception
+ */
+ @Test
+ public void testMapNullKey() throws Exception {
+ assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
+ String pigSchema = "m:map[]";
+
+ String tableSchema = "m map<string, string>";
+
+ List<Tuple> data = new ArrayList<Tuple>();
+ Tuple t = t(
+ new HashMap<String, String>() {
+ {
+ put("ac test1", "test 1");
+ put("ac test2", "test 2");
+ put(null, "test 3");
+ };
+ });
+ data.add(t);
+
+ List<Tuple> result = new ArrayList<Tuple>();
+ t = t(
+ new HashMap<String, String>() {
+ {
+ put("ac test1", "test 1");
+ put("ac test2", "test 2");
+ };
+ });
+ result.add(t);
+
+ verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true);
+ verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false);
+ }
}