You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/11/15 06:48:03 UTC
svn commit: r1542179 - in /hive/trunk/serde/src:
java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java
Author: hashutosh
Date: Fri Nov 15 05:48:02 2013
New Revision: 1542179
URL: http://svn.apache.org/r1542179
Log:
HIVE-5788 : select * fails for table after adding new columns using rcfile storage format (Szehon Ho via Ashutosh Chauhan)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java?rev=1542179&r1=1542178&r2=1542179&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java Fri Nov 15 05:48:02 2013
@@ -67,6 +67,7 @@ public abstract class ColumnarStructBase
if (col != null) {
rawBytesField = col;
inited = false;
+ fieldSkipped = false;
} else {
// select columns that actually do not exist in the file.
fieldSkipped = true;
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java?rev=1542179&r1=1542178&r2=1542179&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java Fri Nov 15 05:48:02 2013
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.SimpleMapEqualComparer;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.LongWritable;
public class TestLazyBinaryColumnarSerDe extends TestCase {
@@ -238,5 +240,86 @@ public class TestLazyBinaryColumnarSerDe
}
}
+ private static class BeforeStruct {
+ Long l1;
+ Long l2;
+ }
+
+ private static class AfterStruct {
+ Long l1;
+ Long l2;
+ Long l3;
+ }
+
+ /**
+ * HIVE-5788
+ * <p>
+ * Background: in cases of "add column", table metadata changes but data does not. Columns
+ * missing from the data but which are required by metadata are interpreted as null.
+ * <p>
+ * This tests the use-case of altering columns of a table with already some data, then adding more data
+ * in the new schema, and seeing if this serde can to read both types of data from the resultant table.
+ * @throws SerDeException
+ */
+ public void testHandlingAlteredSchemas() throws SerDeException {
+ StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(BeforeStruct.class,
+ ObjectInspectorOptions.JAVA);
+ String cols = ObjectInspectorUtils.getFieldNames(oi);
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, cols);
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ ObjectInspectorUtils.getFieldTypes(oi));
+
+ // serialize some data in the schema before it is altered.
+ LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
+ serde.initialize(new Configuration(), props);
+
+ BeforeStruct bs1 = new BeforeStruct();
+ bs1.l1 = 1L;
+ bs1.l2 = 2L;
+ BytesRefArrayWritable braw1 = (BytesRefArrayWritable) serde.serialize(bs1,
+ oi);
+
+ // alter table add column: change the metadata
+ oi = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(AfterStruct.class,
+ ObjectInspectorOptions.JAVA);
+ cols = ObjectInspectorUtils.getFieldNames(oi);
+ props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, cols);
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ ObjectInspectorUtils.getFieldTypes(oi));
+ serde = new LazyBinaryColumnarSerDe();
+ serde.initialize(new Configuration(), props);
-}
+ // serialize some data in the schema after it is altered.
+ AfterStruct as = new AfterStruct();
+ as.l1 = 11L;
+ as.l2 = 12L;
+ as.l3 = 13L;
+ BytesRefArrayWritable braw2 = (BytesRefArrayWritable) serde.serialize(as,
+ oi);
+
+ // fetch operator
+ serde = new LazyBinaryColumnarSerDe();
+ serde.initialize(new Configuration(), props);
+
+ //fetch the row inserted before schema is altered and verify
+ LazyBinaryColumnarStruct struct1 = (LazyBinaryColumnarStruct) serde
+ .deserialize(braw1);
+ oi = (StructObjectInspector) serde.getObjectInspector();
+ List<Object> objs1 = oi.getStructFieldsDataAsList(struct1);
+ Assert.assertEquals(((LongWritable) objs1.get(0)).get(), 1L);
+ Assert.assertEquals(((LongWritable) objs1.get(1)).get(), 2L);
+ Assert.assertNull(objs1.get(2));
+
+ //fetch the row inserted after schema is altered and verify
+ LazyBinaryColumnarStruct struct2 = (LazyBinaryColumnarStruct) serde
+ .deserialize(braw2);
+ List<Object> objs2 = struct2.getFieldsAsList();
+ Assert.assertEquals(((LongWritable) objs2.get(0)).get(), 11L);
+ Assert.assertEquals(((LongWritable) objs2.get(1)).get(), 12L);
+ Assert.assertEquals(((LongWritable) objs2.get(2)).get(), 13L);
+ }
+}
\ No newline at end of file