You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/09/17 08:08:15 UTC
svn commit: r997989 - in /hadoop/hive/trunk: ./ serde/if/test/
serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/
serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/
serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ serde/src/...
Author: namit
Date: Fri Sep 17 06:08:15 2010
New Revision: 997989
URL: http://svn.apache.org/viewvc?rev=997989&view=rev
Log:
HIVE-1616. Add ProtocolsBufferStructObjectInspector
(Johan Oskarsson via namit)
Added:
hadoop/hive/trunk/serde/if/test/complexpb.proto
hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/
hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/
hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=997989&r1=997988&r2=997989&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Sep 17 06:08:15 2010
@@ -71,6 +71,9 @@ Trunk - Unreleased
HIVE-1598. Create an option to specify an option for the format of
intermediate results (Ning Zhang via namit)
+ HIVE-1616. Add ProtocolsBufferStructObjectInspector
+ (Johan Oskarsson via namit)
+
IMPROVEMENTS
HIVE-1394. Do not update transient_lastDdlTime if the partition is modified by a housekeeping
Added: hadoop/hive/trunk/serde/if/test/complexpb.proto
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/if/test/complexpb.proto?rev=997989&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/if/test/complexpb.proto (added)
+++ hadoop/hive/trunk/serde/if/test/complexpb.proto Fri Sep 17 06:08:15 2010
@@ -0,0 +1,15 @@
+package org.apache.hadoop.hive.serde2.proto.test;
+
+message IntString {
+ optional int32 myint = 1;
+ optional string myString = 2;
+ optional int32 underscore_int = 3;
+};
+
+message Complex {
+ optional int32 aint = 1;
+ optional string aString = 2;
+ repeated int32 lint = 3;
+ repeated string lString = 4;
+ repeated IntString lintString = 5;
+};
Added: hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java?rev=997989&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java (added)
+++ hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java Fri Sep 17 06:08:15 2010
@@ -0,0 +1,118 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: complexpb.proto
+
+package org.apache.hadoop.hive.serde2.proto.test;
+
+public final class Complexpb {
+ private Complexpb() {}
+
+ public static final class IntString {
+ public IntString(int myInt, String myString, int underscoreInt) {
+ this.myint_ = myInt;
+ this.hasMyint = true;
+ this.myString_ = myString;
+ this.hasMyString = true;
+ this.underscoreInt_ = underscoreInt;
+ this.hasUnderscoreInt = true;
+ }
+
+ // optional int32 myint = 1;
+ public static final int MYINT_FIELD_NUMBER = 1;
+ private final boolean hasMyint;
+ private int myint_ = 0;
+ public boolean hasMyint() { return hasMyint; }
+ public int getMyint() { return myint_; }
+
+ // optional string myString = 2;
+ public static final int MYSTRING_FIELD_NUMBER = 2;
+ private final boolean hasMyString;
+ private java.lang.String myString_ = "";
+ public boolean hasMyString() { return hasMyString; }
+ public java.lang.String getMyString() { return myString_; }
+
+ // optional int32 underscore_int = 3;
+ public static final int UNDERSCORE_INT_FIELD_NUMBER = 3;
+ private final boolean hasUnderscoreInt;
+ private int underscoreInt_ = 0;
+ public boolean hasUnderscoreInt() { return hasUnderscoreInt; }
+ public int getUnderscoreInt() { return underscoreInt_; }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ }
+
+ public static final class Complex {
+ // Use Complex.newBuilder() to construct.
+ public Complex(int aint, String aString, java.util.List<Integer> lint, java.util.List<String> lString, java.util.List<IntString> lintString) {
+ this.aint_ = aint;
+ this.hasAint = true;
+ this.aString_ = aString;
+ this.hasAString = true;
+ this.lint_ = lint;
+ this.lString_ = lString;
+ this.lintString_ = lintString;
+ }
+
+ // optional int32 aint = 1;
+ public static final int AINT_FIELD_NUMBER = 1;
+ private final boolean hasAint;
+ private int aint_ = 0;
+ public boolean hasAint() { return hasAint; }
+ public int getAint() { return aint_; }
+
+ // optional string aString = 2;
+ public static final int ASTRING_FIELD_NUMBER = 2;
+ private final boolean hasAString;
+ private java.lang.String aString_ = "";
+ public boolean hasAString() { return hasAString; }
+ public java.lang.String getAString() { return aString_; }
+
+ // repeated int32 lint = 3;
+ public static final int LINT_FIELD_NUMBER = 3;
+ private java.util.List<java.lang.Integer> lint_ =
+ java.util.Collections.emptyList();
+ public java.util.List<java.lang.Integer> getLintList() {
+ return lint_;
+ }
+ public int getLintCount() { return lint_.size(); }
+ public int getLint(int index) {
+ return lint_.get(index);
+ }
+
+ // repeated string lString = 4;
+ public static final int LSTRING_FIELD_NUMBER = 4;
+ private java.util.List<java.lang.String> lString_ =
+ java.util.Collections.emptyList();
+ public java.util.List<java.lang.String> getLStringList() {
+ return lString_;
+ }
+ public int getLStringCount() { return lString_.size(); }
+ public java.lang.String getLString(int index) {
+ return lString_.get(index);
+ }
+
+ // repeated .org.apache.hadoop.hive.serde2.proto.test.IntString lintString = 5;
+ public static final int LINTSTRING_FIELD_NUMBER = 5;
+ private java.util.List<org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString> lintString_ =
+ java.util.Collections.emptyList();
+ public java.util.List<org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString> getLintStringList() {
+ return lintString_;
+ }
+ public int getLintStringCount() { return lintString_.size(); }
+ public org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString getLintString(int index) {
+ return lintString_.get(index);
+ }
+
+ private void initFields() {
+ }
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.serde2.proto.test.Complex)
+ }
+
+ // @@protoc_insertion_point(outer_class_scope)
+}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=997989&r1=997988&r2=997989&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java Fri Sep 17 06:08:15 2010
@@ -23,6 +23,7 @@ import java.lang.reflect.GenericArrayTyp
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -48,16 +49,16 @@ public final class ObjectInspectorFactor
/**
* ObjectInspectorOptions describes what ObjectInspector to use. JAVA is to
* use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out
- * __isset fields. New ObjectInspectorOptions can be added here when
- * available.
- *
+ * __isset fields, PROTOCOL_BUFFERS filters out has*.
+ * New ObjectInspectorOptions can be added here when available.
+ *
* We choose to use a single HashMap objectInspectorCache to cache all
* situations for efficiency and code simplicity. And we don't expect a case
* that a user need to create 2 or more different types of ObjectInspectors
* for the same Java type.
*/
public enum ObjectInspectorOptions {
- JAVA, THRIFT
+ JAVA, THRIFT, PROTOCOL_BUFFERS
};
private static HashMap<Type, ObjectInspector> objectInspectorCache = new HashMap<Type, ObjectInspector>();
@@ -69,16 +70,37 @@ public final class ObjectInspectorFactor
oi = getReflectionObjectInspectorNoCache(t, options);
objectInspectorCache.put(t, oi);
}
- if ((options.equals(ObjectInspectorOptions.JAVA) && oi.getClass().equals(
- ThriftStructObjectInspector.class))
- || (options.equals(ObjectInspectorOptions.THRIFT) && oi.getClass()
- .equals(ReflectionStructObjectInspector.class))) {
- throw new RuntimeException(
- "Cannot call getObjectInspectorByReflection with both JAVA and THRIFT !");
- }
+ verifyObjectInspector(options, oi, ObjectInspectorOptions.JAVA, new Class[]{ThriftStructObjectInspector.class,
+ ProtocolBuffersStructObjectInspector.class});
+ verifyObjectInspector(options, oi, ObjectInspectorOptions.THRIFT, new Class[]{ReflectionStructObjectInspector.class,
+ ProtocolBuffersStructObjectInspector.class});
+ verifyObjectInspector(options, oi, ObjectInspectorOptions.PROTOCOL_BUFFERS, new Class[]{ThriftStructObjectInspector.class,
+ ReflectionStructObjectInspector.class});
+
return oi;
}
+ /**
+ * Verify that we don't have an unexpected type of object inspector.
+ * @param option The option to verify
+ * @param oi The ObjectInspector to verify
+ * @param checkOption We're only interested in this option type
+ * @param classes ObjectInspector should not be of these types
+ */
+ private static void verifyObjectInspector(ObjectInspectorOptions option, ObjectInspector oi,
+ ObjectInspectorOptions checkOption, Class[] classes) {
+
+ if (option.equals(checkOption)) {
+ for (Class checkClass : classes) {
+ if (oi.getClass().equals(checkClass)) {
+ throw new RuntimeException(
+ "Cannot call getObjectInspectorByReflection with more then one of " +
+ Arrays.toString(ObjectInspectorOptions.values()) + "!");
+ }
+ }
+ }
+ }
+
private static ObjectInspector getReflectionObjectInspectorNoCache(Type t,
ObjectInspectorOptions options) {
if (t instanceof GenericArrayType) {
@@ -147,6 +169,9 @@ public final class ObjectInspectorFactor
case THRIFT:
oi = new ThriftStructObjectInspector();
break;
+ case PROTOCOL_BUFFERS:
+ oi = new ProtocolBuffersStructObjectInspector();
+ break;
default:
throw new RuntimeException(ObjectInspectorFactory.class.getName()
+ ": internal error.");
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java?rev=997989&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java Fri Sep 17 06:08:15 2010
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ *
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
+ */
+class ProtocolBuffersStructObjectInspector extends ReflectionStructObjectInspector {
+
+ @Override
+ public boolean shouldIgnoreField(String name) {
+ return name.startsWith("has");
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java?rev=997989&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java (added)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java Fri Sep 17 06:08:15 2010
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.proto.test.Complexpb.Complex;
+import org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString;
+
+/**
+ * TestProtocolBuffersObjectInspectors.
+ *
+ */
+public class TestProtocolBuffersObjectInspectors extends TestCase {
+
+ public void testProtocolBuffersObjectInspectors() throws Throwable {
+
+ try {
+ ObjectInspector oi1 = ObjectInspectorFactory
+ .getReflectionObjectInspector(Complex.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
+ ObjectInspector oi2 = ObjectInspectorFactory
+ .getReflectionObjectInspector(Complex.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
+ assertEquals(oi1, oi2);
+
+ // metadata
+ assertEquals(Category.STRUCT, oi1.getCategory());
+ StructObjectInspector soi = (StructObjectInspector) oi1;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ assertEquals(5, fields.size());
+ assertEquals(fields.get(0), soi.getStructFieldRef("aint_"));
+
+ // null
+ for (int i = 0; i < fields.size(); i++) {
+ assertNull(soi.getStructFieldData(null, fields.get(i)));
+ }
+
+ // real object
+ List<Integer> c2 = Arrays.asList(new Integer[] {1, 2, 3});
+ List<String> c3 = Arrays.asList(new String[] {"one", "two"});
+ List<IntString> c4 = new ArrayList<IntString>();
+ Complex c = new Complex(1, "test", c2, c3, c4);
+
+ assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
+ assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
+ assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
+ assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
+ assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
+ ArrayList<Object> cfields = new ArrayList<Object>();
+ for (int i = 0; i < 5; i++) {
+ cfields.add(soi.getStructFieldData(c, fields.get(i)));
+ }
+ assertEquals(cfields, soi.getStructFieldsDataAsList(c));
+
+ // sub fields
+ assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ fields.get(0).getFieldObjectInspector());
+ assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ fields.get(1).getFieldObjectInspector());
+ assertEquals(
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector),
+ fields.get(2).getFieldObjectInspector());
+ assertEquals(
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory
+ .javaStringObjectInspector),
+ fields.get(3).getFieldObjectInspector());
+ assertEquals(ObjectInspectorFactory
+ .getStandardListObjectInspector(ObjectInspectorFactory
+ .getReflectionObjectInspector(IntString.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS)),
+ fields.get(4).getFieldObjectInspector());
+ } catch (Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+}