You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/08/12 21:26:29 UTC
svn commit: r1157224 - in /hive/trunk: ivy/
metastore/src/java/org/apache/hadoop/hive/metastore/ serde/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/
serde/sr...
Author: jvs
Date: Fri Aug 12 19:26:29 2011
New Revision: 1157224
URL: http://svn.apache.org/viewvc?rev=1157224&view=rev
Log:
HIVE-2171. Allow custom serdes to set field comments
(Jakob Homan via jvs)
Added:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
Modified:
hive/trunk/ivy/libraries.properties
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/trunk/serde/ivy.xml
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Fri Aug 12 19:26:29 2011
@@ -44,6 +44,7 @@ junit.version=3.8.1
ivy.version=2.1.0
log4j.version=1.2.15
maven-ant-tasks.version=2.1.0
+mockito-all.version=1.8.2
slf4j-api.version=1.6.1
slf4j-log4j12.version=1.6.1
thrift.version=0.5.0
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri Aug 12 19:26:29 2011
@@ -823,21 +823,27 @@ public class MetaStoreUtils {
// rules on how to recurse the ObjectInspector based on its type
if (oi.getCategory() != Category.STRUCT) {
str_fields.add(new FieldSchema(last_name, oi.getTypeName(),
- "from deserializer"));
+ FROM_SERIALIZER));
} else {
List<? extends StructField> fields = ((StructObjectInspector) oi)
.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
- String fieldName = fields.get(i).getFieldName();
- String fieldTypeName = fields.get(i).getFieldObjectInspector()
- .getTypeName();
- str_fields.add(new FieldSchema(fieldName, fieldTypeName,
- "from deserializer"));
+ StructField structField = fields.get(i);
+ String fieldName = structField.getFieldName();
+ String fieldTypeName = structField.getFieldObjectInspector().getTypeName();
+ String fieldComment = determineFieldComment(structField.getFieldComment());
+
+ str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment));
}
}
return str_fields;
}
+ private static final String FROM_SERIALIZER = "from deserializer";
+ private static String determineFieldComment(String comment) {
+ return (comment == null || comment.isEmpty()) ? FROM_SERIALIZER : comment;
+ }
+
/**
* Convert TypeInfo to FieldSchema.
*/
Modified: hive/trunk/serde/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/serde/ivy.xml?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/ivy.xml (original)
+++ hive/trunk/serde/ivy.xml Fri Aug 12 19:26:29 2011
@@ -28,5 +28,6 @@
</dependency>
<dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}"/>
<dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"/>
+ <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}"/>
</dependencies>
</ivy-module>
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java Fri Aug 12 19:26:29 2011
@@ -46,6 +46,16 @@ public final class LazyObjectInspectorFa
List<ObjectInspector> structFieldObjectInspectors, byte separator,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
byte escapeChar) {
+ return getLazySimpleStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors, null, separator, nullSequence,
+ lastColumnTakesRest, escaped, escapeChar);
+ }
+
+ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
+ byte separator, Text nullSequence, boolean lastColumnTakesRest,
+ boolean escaped,byte escapeChar) {
ArrayList<Object> signature = new ArrayList<Object>();
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
@@ -54,12 +64,15 @@ public final class LazyObjectInspectorFa
signature.add(Boolean.valueOf(lastColumnTakesRest));
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
+ if(structFieldComments != null) {
+ signature.add(structFieldComments);
+ }
LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector
.get(signature);
if (result == null) {
result = new LazySimpleStructObjectInspector(structFieldNames,
- structFieldObjectInspectors, separator, nullSequence,
- lastColumnTakesRest, escaped, escapeChar);
+ structFieldObjectInspectors, structFieldComments, separator,
+ nullSequence, lastColumnTakesRest, escaped, escapeChar);
cachedLazySimpleStructObjectInspector.put(signature, result);
}
return result;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -49,6 +49,7 @@ public class LazySimpleStructObjectInspe
protected int fieldID;
protected String fieldName;
protected ObjectInspector fieldObjectInspector;
+ protected String fieldComment;
public MyField(int fieldID, String fieldName,
ObjectInspector fieldObjectInspector) {
@@ -57,6 +58,11 @@ public class LazySimpleStructObjectInspe
this.fieldObjectInspector = fieldObjectInspector;
}
+ public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector, String fieldComment) {
+ this(fieldID, fieldName, fieldObjectInspector);
+ this.fieldComment = fieldComment;
+ }
+
public int getFieldID() {
return fieldID;
}
@@ -69,6 +75,10 @@ public class LazySimpleStructObjectInspe
return fieldObjectInspector;
}
+ public String getFieldComment() {
+ return fieldComment;
+ }
+
@Override
public String toString() {
return "" + fieldID + ":" + fieldName;
@@ -95,15 +105,26 @@ public class LazySimpleStructObjectInspe
List<ObjectInspector> structFieldObjectInspectors, byte separator,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
byte escapeChar) {
- init(structFieldNames, structFieldObjectInspectors, separator,
+ init(structFieldNames, structFieldObjectInspectors, null, separator,
nullSequence, lastColumnTakesRest, escaped, escapeChar);
}
+ public LazySimpleStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments, byte separator, Text nullSequence,
+ boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
+ init(structFieldNames, structFieldObjectInspectors, structFieldComments,
+ separator, nullSequence, lastColumnTakesRest, escaped, escapeChar);
+ }
+
protected void init(List<String> structFieldNames,
- List<ObjectInspector> structFieldObjectInspectors, byte separator,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments, byte separator,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
byte escapeChar) {
assert (structFieldNames.size() == structFieldObjectInspectors.size());
+ assert (structFieldComments == null ||
+ structFieldNames.size() == structFieldComments.size());
this.separator = separator;
this.nullSequence = nullSequence;
@@ -114,7 +135,8 @@ public class LazySimpleStructObjectInspe
fields = new ArrayList<MyField>(structFieldNames.size());
for (int i = 0; i < structFieldNames.size(); i++) {
fields.add(new MyField(i, structFieldNames.get(i),
- structFieldObjectInspectors.get(i)));
+ structFieldObjectInspectors.get(i),
+ structFieldComments == null ? null : structFieldComments.get(i)));
}
}
@@ -131,7 +153,7 @@ public class LazySimpleStructObjectInspe
this.fields = new ArrayList<MyField>(fields.size());
for (int i = 0; i < fields.size(); i++) {
this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
- .get(i).getFieldObjectInspector()));
+ .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java Fri Aug 12 19:26:29 2011
@@ -42,14 +42,24 @@ public final class LazyBinaryObjectInspe
public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- ArrayList<Object> signature = new ArrayList<Object>();
+ return getLazyBinaryStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors, null);
+ }
+
+ public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments) {
+ ArrayList<Object> signature = new ArrayList<Object>(3);
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
+ if(structFieldComments != null) {
+ signature.add(structFieldComments);
+ }
LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector
.get(signature);
if (result == null) {
result = new LazyBinaryStructObjectInspector(structFieldNames,
- structFieldObjectInspectors);
+ structFieldObjectInspectors, structFieldComments);
cachedLazyBinaryStructObjectInspector.put(signature, result);
}
return result;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -37,6 +37,12 @@ public class LazyBinaryStructObjectInspe
super(structFieldNames, structFieldObjectInspectors);
}
+ protected LazyBinaryStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
+ super(structFieldNames, structFieldObjectInspectors, structFieldComments);
+ }
+
protected LazyBinaryStructObjectInspector(List<StructField> fields) {
super(fields);
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -44,6 +44,7 @@ class ColumnarStructObjectInspector exte
protected int fieldID;
protected String fieldName;
protected ObjectInspector fieldObjectInspector;
+ protected String fieldComment;
public MyField(int fieldID, String fieldName,
ObjectInspector fieldObjectInspector) {
@@ -52,6 +53,12 @@ class ColumnarStructObjectInspector exte
this.fieldObjectInspector = fieldObjectInspector;
}
+ public MyField(int fieldID, String fieldName,
+ ObjectInspector fieldObjectInspector, String fieldComment) {
+ this(fieldID, fieldName, fieldObjectInspector);
+ this.fieldComment = fieldComment;
+ }
+
public int getFieldID() {
return fieldID;
}
@@ -64,6 +71,9 @@ class ColumnarStructObjectInspector exte
return fieldObjectInspector;
}
+ public String getFieldComment() {
+ return fieldComment;
+ }
@Override
public String toString() {
return "" + fieldID + ":" + fieldName;
@@ -82,17 +92,27 @@ class ColumnarStructObjectInspector exte
*/
public ColumnarStructObjectInspector(List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- init(structFieldNames, structFieldObjectInspectors);
+ init(structFieldNames, structFieldObjectInspectors, null);
+ }
+
+ public ColumnarStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
+ init(structFieldNames, structFieldObjectInspectors, structFieldComments);
}
protected void init(List<String> structFieldNames,
- List<ObjectInspector> structFieldObjectInspectors) {
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
assert (structFieldNames.size() == structFieldObjectInspectors.size());
+ assert (structFieldComments == null ||
+ (structFieldNames.size() == structFieldComments.size()));
fields = new ArrayList<MyField>(structFieldNames.size());
for (int i = 0; i < structFieldNames.size(); i++) {
fields.add(new MyField(i, structFieldNames.get(i),
- structFieldObjectInspectors.get(i)));
+ structFieldObjectInspectors.get(i),
+ structFieldComments == null ? null : structFieldComments.get(i)));
}
}
@@ -104,7 +124,7 @@ class ColumnarStructObjectInspector exte
this.fields = new ArrayList<MyField>(fields.size());
for (int i = 0; i < fields.size(); i++) {
this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
- .get(i).getFieldObjectInspector()));
+ .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.serde2.objectinspector;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -38,17 +39,33 @@ import org.apache.hadoop.hive.serde2.obj
public class MetadataListStructObjectInspector extends
StandardStructObjectInspector {
- static HashMap<List<String>, MetadataListStructObjectInspector> cached = new HashMap<List<String>, MetadataListStructObjectInspector>();
+ static HashMap<List<List<String>>, MetadataListStructObjectInspector>
+ cached = new HashMap<List<List<String>>, MetadataListStructObjectInspector>();
// public static MetadataListStructObjectInspector getInstance(int fields) {
// return getInstance(ObjectInspectorUtils.getIntegerArray(fields));
// }
public static MetadataListStructObjectInspector getInstance(
List<String> columnNames) {
+ ArrayList<List<String>> key = new ArrayList<List<String>>(1);
+ key.add(columnNames);
MetadataListStructObjectInspector result = cached.get(columnNames);
if (result == null) {
result = new MetadataListStructObjectInspector(columnNames);
- cached.put(columnNames, result);
+ cached.put(key, result);
+ }
+ return result;
+ }
+
+ public static MetadataListStructObjectInspector getInstance(
+ List<String> columnNames, List<String> columnComments) {
+ ArrayList<List<String>> key = new ArrayList<List<String>>(2);
+ Collections.addAll(key, columnNames, columnComments);
+
+ MetadataListStructObjectInspector result = cached.get(key);
+ if (result == null) {
+ result = new MetadataListStructObjectInspector(columnNames, columnComments);
+ cached.put(key, result);
}
return result;
}
@@ -66,6 +83,12 @@ public class MetadataListStructObjectIns
super(columnNames, getFieldObjectInspectors(columnNames.size()));
}
+ public MetadataListStructObjectInspector(List<String> columnNames,
+ List<String> columnComments) {
+ super(columnNames, getFieldObjectInspectors(columnNames.size()),
+ columnComments);
+ }
+
// Get col object out
@Override
public Object getStructFieldData(Object data, StructField fieldRef) {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java Fri Aug 12 19:26:29 2011
@@ -242,14 +242,22 @@ public final class ObjectInspectorFactor
public static StandardStructObjectInspector getStandardStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- ArrayList<List<?>> signature = new ArrayList<List<?>>();
+ return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, null);
+ }
+
+ public static StandardStructObjectInspector getStandardStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structComments) {
+ ArrayList<List<?>> signature = new ArrayList<List<?>>(3);
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
- StandardStructObjectInspector result = cachedStandardStructObjectInspector
- .get(signature);
- if (result == null) {
- result = new StandardStructObjectInspector(structFieldNames,
- structFieldObjectInspectors);
+ if(structComments != null) {
+ signature.add(structComments);
+ }
+ StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature);
+ if(result == null) {
+ result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, structComments);
cachedStandardStructObjectInspector.put(signature, result);
}
return result;
@@ -273,14 +281,23 @@ public final class ObjectInspectorFactor
public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- ArrayList<Object> signature = new ArrayList<Object>();
+ return getColumnarStructObjectInspector(structFieldNames, structFieldObjectInspectors, null);
+ }
+
+ public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments) {
+ ArrayList<Object> signature = new ArrayList<Object>(3);
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
+ if(structFieldComments != null) {
+ signature.add(structFieldComments);
+ }
ColumnarStructObjectInspector result = cachedColumnarStructObjectInspector
.get(signature);
if (result == null) {
result = new ColumnarStructObjectInspector(structFieldNames,
- structFieldObjectInspectors);
+ structFieldObjectInspectors, structFieldComments);
cachedColumnarStructObjectInspector.put(signature, result);
}
return result;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -58,6 +58,10 @@ public class ReflectionStructObjectInspe
return fieldObjectInspector;
}
+ public String getFieldComment() {
+ return null;
+ }
+
@Override
public String toString() {
return field.toString();
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -46,6 +46,7 @@ public class StandardStructObjectInspect
protected int fieldID;
protected String fieldName;
protected ObjectInspector fieldObjectInspector;
+ protected String fieldComment;
public MyField(int fieldID, String fieldName,
ObjectInspector fieldObjectInspector) {
@@ -54,6 +55,12 @@ public class StandardStructObjectInspect
this.fieldObjectInspector = fieldObjectInspector;
}
+ public MyField(int fieldID, String fieldName,
+ ObjectInspector fieldObjectInspector, String fieldComment) {
+ this(fieldID, fieldName, fieldObjectInspector);
+ this.fieldComment = fieldComment;
+ }
+
public int getFieldID() {
return fieldID;
}
@@ -66,6 +73,10 @@ public class StandardStructObjectInspect
return fieldObjectInspector;
}
+ public String getFieldComment() {
+ return fieldComment;
+ }
+
@Override
public String toString() {
return "" + fieldID + ":" + fieldName;
@@ -83,17 +94,30 @@ public class StandardStructObjectInspect
*/
protected StandardStructObjectInspector(List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- init(structFieldNames, structFieldObjectInspectors);
+ init(structFieldNames, structFieldObjectInspectors, null);
+ }
+
+ /**
+ * Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+ */
+ protected StandardStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
+ init(structFieldNames, structFieldObjectInspectors, structFieldComments);
}
protected void init(List<String> structFieldNames,
- List<ObjectInspector> structFieldObjectInspectors) {
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
assert (structFieldNames.size() == structFieldObjectInspectors.size());
+ assert (structFieldComments == null ||
+ (structFieldNames.size() == structFieldComments.size()));
fields = new ArrayList<MyField>(structFieldNames.size());
for (int i = 0; i < structFieldNames.size(); i++) {
fields.add(new MyField(i, structFieldNames.get(i),
- structFieldObjectInspectors.get(i)));
+ structFieldObjectInspectors.get(i),
+ structFieldComments == null ? null : structFieldComments.get(i)));
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java Fri Aug 12 19:26:29 2011
@@ -36,4 +36,8 @@ public interface StructField {
*/
ObjectInspector getFieldObjectInspector();
+ /**
+ * Get the comment for the field. May be null if no comment provided.
+ */
+ String getFieldComment();
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java Fri Aug 12 19:26:29 2011
@@ -55,6 +55,10 @@ public class UnionStructObjectInspector
public ObjectInspector getFieldObjectInspector() {
return structField.getFieldObjectInspector();
}
+
+ public String getFieldComment() {
+ return structField.getFieldComment();
+ }
}
List<StructObjectInspector> unionObjectInspectors;
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java?rev=1157224&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java Fri Aug 12 19:26:29 2011
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestSerdeWithFieldComments extends TestCase {
+
+ private StructField mockedStructField(String name, String oiTypeName,
+ String comment) {
+ StructField m = mock(StructField.class);
+ when(m.getFieldName()).thenReturn(name);
+
+ ObjectInspector oi = mock(ObjectInspector.class);
+ when(oi.getTypeName()).thenReturn(oiTypeName);
+
+ when(m.getFieldObjectInspector()).thenReturn(oi);
+ when(m.getFieldComment()).thenReturn(comment);
+
+ return m;
+ }
+
+ public void testFieldComments() throws MetaException, SerDeException {
+ StructObjectInspector mockSOI = mock(StructObjectInspector.class);
+ when(mockSOI.getCategory()).thenReturn(ObjectInspector.Category.STRUCT);
+ List fieldRefs = new ArrayList<StructField>();
+ // Add field with a comment...
+ fieldRefs.add(mockedStructField("first", "type name 1", "this is a comment"));
+ // ... and one without
+ fieldRefs.add(mockedStructField("second", "type name 2", null));
+
+ when(mockSOI.getAllStructFieldRefs()).thenReturn(fieldRefs);
+
+ Deserializer mockDe = mock(Deserializer.class);
+ when(mockDe.getObjectInspector()).thenReturn(mockSOI);
+ List<FieldSchema> result =
+ MetaStoreUtils.getFieldsFromDeserializer("testTable", mockDe);
+
+ assertEquals(2, result.size());
+ assertEquals("first", result.get(0).getName());
+ assertEquals("this is a comment", result.get(0).getComment());
+ assertEquals("second", result.get(1).getName());
+ assertEquals("from deserializer", result.get(1).getComment());
+ }
+}
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java Fri Aug 12 19:26:29 2011
@@ -263,75 +263,9 @@ public class TestStandardObjectInspector
@SuppressWarnings("unchecked")
public void testStandardStructObjectInspector() throws Throwable {
try {
- ArrayList<String> fieldNames = new ArrayList<String>();
- fieldNames.add("firstInteger");
- fieldNames.add("secondString");
- fieldNames.add("thirdBoolean");
- ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
- fieldObjectInspectors
- .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
- fieldObjectInspectors
- .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- fieldObjectInspectors
- .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
-
- StandardStructObjectInspector soi1 = ObjectInspectorFactory
- .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
- StandardStructObjectInspector soi2 = ObjectInspectorFactory
- .getStandardStructObjectInspector((ArrayList<String>) fieldNames
- .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
- .clone());
- assertEquals(soi1, soi2);
-
- // metadata
- assertEquals(Category.STRUCT, soi1.getCategory());
- List<? extends StructField> fields = soi1.getAllStructFieldRefs();
- assertEquals(3, fields.size());
- for (int i = 0; i < 3; i++) {
- assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i)
- .getFieldName());
- assertEquals(fieldObjectInspectors.get(i), fields.get(i)
- .getFieldObjectInspector());
- }
- assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
- StringBuilder structTypeName = new StringBuilder();
- structTypeName.append("struct<");
- for (int i = 0; i < fields.size(); i++) {
- if (i > 0) {
- structTypeName.append(",");
- }
- structTypeName.append(fields.get(i).getFieldName());
- structTypeName.append(":");
- structTypeName.append(fields.get(i).getFieldObjectInspector()
- .getTypeName());
- }
- structTypeName.append(">");
- assertEquals(structTypeName.toString(), soi1.getTypeName());
-
- // null
- assertNull(soi1.getStructFieldData(null, fields.get(0)));
- assertNull(soi1.getStructFieldData(null, fields.get(1)));
- assertNull(soi1.getStructFieldData(null, fields.get(2)));
- assertNull(soi1.getStructFieldsDataAsList(null));
-
- // HashStruct
- ArrayList<Object> struct = new ArrayList<Object>(3);
- struct.add(1);
- struct.add("two");
- struct.add(true);
-
- assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
- assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
- assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
-
- // Settable
- Object struct3 = soi1.create();
- System.out.println(struct3);
- soi1.setStructFieldData(struct3, fields.get(0), 1);
- soi1.setStructFieldData(struct3, fields.get(1), "two");
- soi1.setStructFieldData(struct3, fields.get(2), true);
- assertEquals(struct, struct3);
-
+ // Test StandardObjectInspector both with field comments and without
+ doStandardObjectInspectorTest(true);
+ doStandardObjectInspectorTest(false);
} catch (Throwable e) {
e.printStackTrace();
throw e;
@@ -339,6 +273,95 @@ public class TestStandardObjectInspector
}
+ private void doStandardObjectInspectorTest(boolean testComments) {
+ ArrayList<String> fieldNames = new ArrayList<String>();
+ fieldNames.add("firstInteger");
+ fieldNames.add("secondString");
+ fieldNames.add("thirdBoolean");
+ ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
+ ArrayList<String> fieldComments = new ArrayList<String>(3);
+ if(testComments) {
+ fieldComments.add("firstInteger comment");
+ fieldComments.add("secondString comment");
+ fieldComments.add("thirdBoolean comment");
+ } else { // should have null for non-specified comments
+ for(int i = 0; i < 3; i++) fieldComments.add(null);
+ }
+
+ StandardStructObjectInspector soi1 = testComments ?
+ ObjectInspectorFactory
+ .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors,
+ fieldComments)
+ : ObjectInspectorFactory
+ .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
+ StandardStructObjectInspector soi2 = testComments ?
+ ObjectInspectorFactory
+ .getStandardStructObjectInspector((ArrayList<String>) fieldNames
+ .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
+ .clone(), (ArrayList<String>)fieldComments.clone())
+ : ObjectInspectorFactory
+ .getStandardStructObjectInspector((ArrayList<String>) fieldNames
+ .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
+ .clone());
+ assertEquals(soi1, soi2);
+
+ // metadata
+ assertEquals(Category.STRUCT, soi1.getCategory());
+ List<? extends StructField> fields = soi1.getAllStructFieldRefs();
+ assertEquals(3, fields.size());
+ for (int i = 0; i < 3; i++) {
+ assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i)
+ .getFieldName());
+ assertEquals(fieldObjectInspectors.get(i), fields.get(i)
+ .getFieldObjectInspector());
+ assertEquals(fieldComments.get(i), fields.get(i).getFieldComment());
+ }
+ assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
+ StringBuilder structTypeName = new StringBuilder();
+ structTypeName.append("struct<");
+ for (int i = 0; i < fields.size(); i++) {
+ if (i > 0) {
+ structTypeName.append(",");
+ }
+ structTypeName.append(fields.get(i).getFieldName());
+ structTypeName.append(":");
+ structTypeName.append(fields.get(i).getFieldObjectInspector()
+ .getTypeName());
+ }
+ structTypeName.append(">");
+ assertEquals(structTypeName.toString(), soi1.getTypeName());
+
+ // null
+ assertNull(soi1.getStructFieldData(null, fields.get(0)));
+ assertNull(soi1.getStructFieldData(null, fields.get(1)));
+ assertNull(soi1.getStructFieldData(null, fields.get(2)));
+ assertNull(soi1.getStructFieldsDataAsList(null));
+
+ // HashStruct
+ ArrayList<Object> struct = new ArrayList<Object>(3);
+ struct.add(1);
+ struct.add("two");
+ struct.add(true);
+
+ assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
+ assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
+ assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
+
+ // Settable
+ Object struct3 = soi1.create();
+ System.out.println(struct3);
+ soi1.setStructFieldData(struct3, fields.get(0), 1);
+ soi1.setStructFieldData(struct3, fields.get(1), "two");
+ soi1.setStructFieldData(struct3, fields.get(2), true);
+ assertEquals(struct, struct3);
+ }
+
@SuppressWarnings("unchecked")
public void testStandardUnionObjectInspector() throws Throwable {
try {