You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/11/30 01:11:55 UTC
[58/91] [abbrv] hive git commit: HIVE-12406: HIVE-9500 introduced
incompatible change to LazySimpleSerDe public interface (Aihua Xu,
reviewed by Szehon Ho)
HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua Xu, reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/306a6402
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/306a6402
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/306a6402
Branch: refs/heads/spark
Commit: 306a640241bf22b97ed546c2a17a7e90e5bd73d6
Parents: 0c1de97
Author: Aihua Xu <ai...@apache.org>
Authored: Mon Nov 23 14:41:58 2015 -0500
Committer: Aihua Xu <ai...@apache.org>
Committed: Mon Nov 23 15:18:42 2015 -0500
----------------------------------------------------------------------
.../hive/serde2/lazy/LazySerDeParameters.java | 6 ++-
.../hive/serde2/lazy/LazySimpleSerDe.java | 38 +++++++++----
.../hive/serde2/lazy/TestLazySimpleSerDe.java | 56 +++++++++++++++++---
3 files changed, 82 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
index 54f6b2b..11af860 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.hive.serde2.lazy;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -42,6 +42,8 @@ import org.apache.hive.common.util.HiveStringUtils;
* SerDeParameters.
*
*/
+@Public
+@Stable
public class LazySerDeParameters implements LazyObjectInspectorParameters {
public static final Logger LOG = LoggerFactory.getLogger(LazySerDeParameters.class.getName());
public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
index 0a2f44c..ac2d39f 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
@@ -19,16 +19,14 @@
package org.apache.hadoop.hive.serde2.lazy;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe;
import org.apache.hadoop.hive.serde2.ByteStream;
@@ -36,7 +34,6 @@ import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
@@ -48,14 +45,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BinaryComparable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hive.common.util.HiveStringUtils;
-
/**
* LazySimpleSerDe can be used to read the same data format as
@@ -67,6 +60,8 @@ import org.apache.hive.common.util.HiveStringUtils;
* Also LazySimpleSerDe outputs typed columns instead of treating all columns as
* String like MetadataTypedColumnsetSerDe.
*/
+@Public
+@Stable
@SerDeSpec(schemaProps = {
serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM,
@@ -413,4 +408,29 @@ public class LazySimpleSerDe extends AbstractEncodingAwareSerDe {
Text text = (Text)blob;
return SerDeUtils.transformTextToUTF8(text, this.charset);
}
+
+ /**
+ * This method is deprecated and is only used for backward compatibility.
+ * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration, Properties, String)
+ */
+ @Deprecated
+ public static SerDeParameters initSerdeParams(Configuration job,
+ Properties tbl,
+ String serdeName) throws SerDeException {
+ return new SerDeParameters(job, tbl, serdeName);
+ }
+
+ /**
+ * This class is deprecated and is only used for backward compatibility. Replace by
+ * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.
+ */
+ @Deprecated
+ public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters {
+
+ public SerDeParameters(Configuration job,
+ Properties tbl,
+ String serdeName) throws SerDeException {
+ super(job, tbl, serdeName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
index b11ce32..391edd4 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.lazy;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
+import java.util.Random;
import junit.framework.TestCase;
@@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
/**
* TestLazySimpleSerDe.
@@ -81,8 +89,6 @@ public class TestLazySimpleSerDe extends TestCase {
}
}
-
-
/**
* Test the LazySimpleSerDe class with LastColumnTakesRest option.
*/
@@ -167,20 +173,56 @@ public class TestLazySimpleSerDe extends TestCase {
throw e;
}
}
-
- Object serializeAndDeserialize(List<Integer> o1, StructObjectInspector oi1,
+
+ /**
+ * Tests the deprecated usage of SerDeParameters.
+ *
+ */
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testSerDeParameters() throws SerDeException, IOException {
+ // Setup
+ LazySimpleSerDe serDe = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+
+ MyTestClass row = new MyTestClass();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ row.randomFill(new Random(1234), extraTypeInfo);
+
+ StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(MyTestClass.class,
+ ObjectInspectorOptions.JAVA);
+
+ String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+ String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+ Properties schema = new Properties();
+ schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
+ schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
+
+ SerDeUtils.initializeSerDe(serDe, conf, schema, null);
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName");
+
+ // Test
+ LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams);
+ assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get());
+ assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get());
+ }
+
+ private Object serializeAndDeserialize(Object row,
+ StructObjectInspector rowOI,
LazySimpleSerDe serde,
LazySerDeParameters serdeParams) throws IOException, SerDeException {
ByteStream.Output serializeStream = new ByteStream.Output();
- LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams
.getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
.isEscaped(), serdeParams.getEscapeChar(), serdeParams
.getNeedsEscape());
+
Text t = new Text(serializeStream.toByteArray());
return serde.deserialize(t);
}
-
-
+
private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s,
Object[] expectedFieldsData) throws SerDeException {
// Get the row structure