You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ti...@apache.org on 2015/04/28 00:38:55 UTC
parquet-mr git commit: Revert "PARQUET-252 : Support nests container
types for scrooge support"
Repository: parquet-mr
Updated Branches:
refs/heads/master f28aa7104 -> 720b988fd
Revert "PARQUET-252 : Support nests container types for scrooge support"
This reverts commit f28aa71041181867a720134e26b64b03cbccb6ec.
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/720b988f
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/720b988f
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/720b988f
Branch: refs/heads/master
Commit: 720b988fd8d7a50fbe922e6c73a3681b1c566331
Parents: f28aa71
Author: Tianshuo Deng <td...@twitter.com>
Authored: Mon Apr 27 15:37:21 2015 -0700
Committer: Tianshuo Deng <td...@twitter.com>
Committed: Mon Apr 27 15:37:21 2015 -0700
----------------------------------------------------------------------
.../parquet/scrooge/ScroogeStructConverter.java | 316 ++++++++-----------
.../scrooge/ScroogeStructConverterTest.java | 151 +++------
parquet-scrooge/src/test/thrift/test.thrift | 93 +-----
3 files changed, 188 insertions(+), 372 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java b/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
index 2c3b761..c5bb72f 100644
--- a/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
+++ b/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
@@ -18,31 +18,24 @@
*/
package parquet.scrooge;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.ParameterizedType;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import scala.collection.JavaConversions;
-import scala.collection.JavaConversions$;
-import scala.collection.Seq;
-import scala.reflect.Manifest;
-
import com.twitter.scrooge.ThriftStructCodec;
import com.twitter.scrooge.ThriftStructFieldInfo;
-
import parquet.thrift.struct.ThriftField;
import parquet.thrift.struct.ThriftType;
import parquet.thrift.struct.ThriftType.StructType.StructOrUnionType;
import parquet.thrift.struct.ThriftTypeID;
-import static parquet.thrift.struct.ThriftField.Requirement;
-import static parquet.thrift.struct.ThriftField.Requirement.DEFAULT;
-import static parquet.thrift.struct.ThriftField.Requirement.REQUIRED;
-import static parquet.thrift.struct.ThriftField.Requirement.OPTIONAL;
+import scala.collection.JavaConversions;
+import scala.collection.JavaConversions$;
+import scala.collection.Seq;
+import scala.reflect.Manifest;
+import java.lang.reflect.*;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import static parquet.thrift.struct.ThriftField.Requirement;
+import static parquet.thrift.struct.ThriftField.Requirement.*;
/**
* Class to convert a scrooge generated class to {@link ThriftType.StructType}. {@link ScroogeReadSupport } uses this
@@ -54,30 +47,18 @@ public class ScroogeStructConverter {
/**
* convert a given scrooge generated class to {@link ThriftType.StructType}
+ *
+ * @param scroogeClass
+ * @return
+ * @throws Exception
*/
public ThriftType.StructType convert(Class scroogeClass) {
return convertStructFromClass(scroogeClass);
}
- private static String mapKeyName(String fieldName) {
- return fieldName + "_map_key";
- }
-
- private static String mapValueName(String fieldName) {
- return fieldName + "_map_value";
- }
-
- private static String listElemName(String fieldName) {
- return fieldName + "_list_elem";
- }
-
- private static String setElemName(String fieldName) {
- return fieldName + "_set_elem";
- }
-
private Class getCompanionClass(Class klass) {
try {
- return Class.forName(klass.getName() + "$");
+ return Class.forName(klass.getName() + "$");
} catch (ClassNotFoundException e) {
throw new ScroogeSchemaConversionException("Can not find companion object for scrooge class " + klass, e);
}
@@ -88,9 +69,9 @@ public class ScroogeStructConverter {
}
private ThriftType.StructType convertCompanionClassToStruct(Class<?> companionClass) {
- ThriftStructCodec<?> companionObject;
+ ThriftStructCodec companionObject = null;
try {
- companionObject = (ThriftStructCodec<?>) companionClass.getField("MODULE$").get(null);
+ companionObject = (ThriftStructCodec<?>)companionClass.getField("MODULE$").get(null);
} catch (ReflectiveOperationException e) {
throw new ScroogeSchemaConversionException("Can not get ThriftStructCodec from companion object of " + companionClass.getName(), e);
}
@@ -107,9 +88,9 @@ public class ScroogeStructConverter {
return new ThriftType.StructType(children, structOrUnionType);
}
- private Iterable<ThriftStructFieldInfo> getFieldInfos(ThriftStructCodec<?> c) {
+ private Iterable<ThriftStructFieldInfo> getFieldInfos(ThriftStructCodec c) {
Class<? extends ThriftStructCodec> klass = c.getClass();
- if (isUnion(klass)) {
+ if (isUnion(klass)){
// Union needs special treatment since currently scrooge does not generates the fieldInfos
// field in the parent union class
return getFieldInfosForUnion(klass);
@@ -117,14 +98,9 @@ public class ScroogeStructConverter {
//each struct has a generated fieldInfos method to provide metadata to its fields
try {
Object r = klass.getMethod("fieldInfos").invoke(c);
- return JavaConversions$.MODULE$.asJavaIterable((scala.collection.Iterable<ThriftStructFieldInfo>) r);
- } catch (ClassCastException e) {
- throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(), e);
- } catch (InvocationTargetException e) {
- throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(), e);
- } catch (NoSuchMethodException e) {
- throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(), e);
- } catch (IllegalAccessException e) {
+ Iterable<ThriftStructFieldInfo> a = JavaConversions$.MODULE$.asJavaIterable((scala.collection.Iterable<ThriftStructFieldInfo>)r);
+ return a;
+ } catch (ReflectiveOperationException e) {
throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(), e);
}
}
@@ -133,25 +109,57 @@ public class ScroogeStructConverter {
private Iterable<ThriftStructFieldInfo> getFieldInfosForUnion(Class klass) {
ArrayList<ThriftStructFieldInfo> fields = new ArrayList<ThriftStructFieldInfo>();
- for (Field f : klass.getDeclaredFields()) {
- if (f.getType().equals(Manifest.class)) {
- Class unionClass = (Class) ((ParameterizedType) f.getGenericType()).getActualTypeArguments()[0];
- Class companionUnionClass = getCompanionClass(unionClass);
- try {
- Object companionUnionObj = companionUnionClass.getField("MODULE$").get(null);
- ThriftStructFieldInfo info = (ThriftStructFieldInfo) companionUnionClass.getMethod("fieldInfo").invoke(companionUnionObj);
- fields.add(info);
- } catch (ReflectiveOperationException e) {
- throw new ScroogeSchemaConversionException("can not find fiedInfo for " + unionClass, e);
- }
+ for(Field f: klass.getDeclaredFields()){
+ if (f.getType().equals(Manifest.class)) {
+ Class unionClass = (Class)((ParameterizedType)f.getGenericType()).getActualTypeArguments()[0];
+ Class companionUnionClass = getCompanionClass(unionClass);
+ try {
+ Object companionUnionObj = companionUnionClass.getField("MODULE$").get(null);
+ ThriftStructFieldInfo info = (ThriftStructFieldInfo)companionUnionClass.getMethod("fieldInfo").invoke(companionUnionObj);
+ fields.add(info);
+ } catch (ReflectiveOperationException e) {
+ throw new ScroogeSchemaConversionException("can not find fiedInfo for " + unionClass, e);
+ }
}
}
return fields;
}
+ private boolean isUnion(Class klass){
+ for(Field f: klass.getDeclaredFields()) {
+ if (f.getName().equals("Union"))
+ return true;
+ }
+ return false;
+ }
+
+
+ private Requirement getRequirementType(ThriftStructFieldInfo f) {
+ if (f.isOptional() && !f.isRequired()) {
+ return OPTIONAL;
+ } else if (f.isRequired() && !f.isOptional()) {
+ return REQUIRED;
+ } else if (!f.isOptional() && !f.isRequired()) {
+ return DEFAULT;
+ } else {
+ throw new ScroogeSchemaConversionException("can not determine requirement type for : " + f.toString()
+ + ", isOptional=" + f.isOptional() + ", isRequired=" + f.isRequired());
+ }
+ }
/**
- * Convert a field in scrooge to ThriftField in parquet
+ * Convert thrift field in scrooge to ThriftField in parquet
+ * Use reflection to detect if a field is optional or required since scrooge does not provide requirement information
+ * in generated classes.
+ * This will not correctly recognize fields that are not specified with a requirement type eg.
+ * struct Address {
+ * 1: string street
+ * }
+ * street will be identified as "REQUIRED"
+ *
+ * @param scroogeField
+ * @return
+ * @throws Exception
*/
public ThriftField toThriftField(ThriftStructFieldInfo scroogeField) {
Requirement requirement = getRequirementType(scroogeField);
@@ -161,87 +169,70 @@ public class ScroogeStructConverter {
ThriftTypeID typeId = ThriftTypeID.fromByte(thriftTypeByte);
ThriftType thriftType;
switch (typeId) {
- case BOOL:
- thriftType = new ThriftType.BoolType();
- break;
- case BYTE:
- thriftType = new ThriftType.ByteType();
- break;
- case DOUBLE:
- thriftType = new ThriftType.DoubleType();
- break;
- case I16:
- thriftType = new ThriftType.I16Type();
- break;
- case I32:
- thriftType = new ThriftType.I32Type();
- break;
- case I64:
- thriftType = new ThriftType.I64Type();
- break;
- case STRING:
- thriftType = new ThriftType.StringType();
- break;
- case STRUCT:
- thriftType = convertStructTypeField(scroogeField);
- break;
- case MAP:
- thriftType = convertMapTypeField(scroogeField, requirement);
- break;
- case SET:
- thriftType = convertSetTypeField(scroogeField, requirement);
- break;
- case LIST:
- thriftType = convertListTypeField(scroogeField, requirement);
- break;
- case ENUM:
- thriftType = convertEnumTypeField(scroogeField);
- break;
- case STOP:
- case VOID:
- default:
- throw new IllegalArgumentException("can't convert type " + typeId);
+ case BOOL:
+ thriftType = new ThriftType.BoolType();
+ break;
+ case BYTE:
+ thriftType = new ThriftType.ByteType();
+ break;
+ case DOUBLE:
+ thriftType = new ThriftType.DoubleType();
+ break;
+ case I16:
+ thriftType = new ThriftType.I16Type();
+ break;
+ case I32:
+ thriftType = new ThriftType.I32Type();
+ break;
+ case I64:
+ thriftType = new ThriftType.I64Type();
+ break;
+ case STRING:
+ thriftType = new ThriftType.StringType();
+ break;
+ case STRUCT:
+ thriftType = convertStructTypeField(scroogeField);
+ break;
+ case MAP:
+ thriftType = convertMapTypeField(scroogeField, requirement);
+ break;
+ case SET:
+ thriftType = convertSetTypeField(scroogeField, requirement);
+ break;
+ case LIST:
+ thriftType = convertListTypeField(scroogeField, requirement);
+ break;
+ case ENUM:
+ thriftType = convertEnumTypeField(scroogeField);
+ break;
+ case STOP:
+ case VOID:
+ default:
+ throw new IllegalArgumentException("can't convert type " + typeId);
}
return new ThriftField(fieldName, fieldId, requirement, thriftType);
}
private ThriftType convertSetTypeField(ThriftStructFieldInfo f, Requirement requirement) {
- return convertSetTypeField(f.tfield().name, f.valueManifest().get(), requirement);
- }
-
- private ThriftType convertSetTypeField(String fieldName, Manifest<?> valueManifest, Requirement requirement) {
- String elemName = setElemName(fieldName);
- ThriftType elementType = convertClassToThriftType(elemName, requirement, valueManifest);
+ ThriftType elementType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
//Set only has one sub-field as element field, therefore using hard-coded 1 as fieldId,
//it's the same as the solution used in ElephantBird
- ThriftField elementField = generateFieldWithoutId(elemName, requirement, elementType);
+ ThriftField elementField = generateFieldWithoutId(f.tfield().name, requirement, elementType);
return new ThriftType.SetType(elementField);
}
private ThriftType convertListTypeField(ThriftStructFieldInfo f, Requirement requirement) {
- return convertListTypeField(f.tfield().name, f.valueManifest().get(), requirement);
- }
-
- private ThriftType convertListTypeField(String fieldName, Manifest<?> valueManifest, Requirement requirement) {
- String elemName = listElemName(fieldName);
- ThriftType elementType = convertClassToThriftType(elemName, requirement, valueManifest);
- ThriftField elementField = generateFieldWithoutId(elemName, requirement, elementType);
+ ThriftType elementType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
+ ThriftField elementField = generateFieldWithoutId(f.tfield().name, requirement, elementType);
return new ThriftType.ListType(elementField);
}
private ThriftType convertMapTypeField(ThriftStructFieldInfo f, Requirement requirement) {
- return convertMapTypeField(f.tfield().name, f.keyManifest().get(), f.valueManifest().get(), requirement);
- }
+ ThriftType keyType = convertClassToThriftType(f.keyManifest().get().runtimeClass());
+ ThriftField keyField = generateFieldWithoutId(f.tfield().name + "_map_key", requirement, keyType);
- private ThriftType convertMapTypeField(String fieldName, Manifest<?> keyManifest, Manifest<?> valueManifest, Requirement requirement) {
-
- String keyName = mapKeyName(fieldName);
- String valueName = mapValueName(fieldName);
- ThriftType keyType = convertClassToThriftType(keyName, requirement, keyManifest);
- ThriftField keyField = generateFieldWithoutId(keyName, requirement, keyType);
-
- ThriftType valueType = convertClassToThriftType(valueName, requirement, valueManifest);
- ThriftField valueField = generateFieldWithoutId(valueName, requirement, valueType);
+ ThriftType valueType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
+ ThriftField valueField = generateFieldWithoutId(f.tfield().name + "_map_value", requirement, valueType);
return new ThriftType.MapType(keyField, valueField);
}
@@ -249,20 +240,26 @@ public class ScroogeStructConverter {
/**
* Generate artificial field, this kind of fields do not have a field ID.
* To be consistent with the behavior in ElephantBird, here uses 1 as the field ID
+ *
+ * @param fieldName
+ * @param requirement
+ * @param thriftType
+ * @return
*/
private ThriftField generateFieldWithoutId(String fieldName, Requirement requirement, ThriftType thriftType) {
- return new ThriftField(fieldName, (short) 1, requirement, thriftType);
+ return new ThriftField(fieldName, (short)1, requirement, thriftType);
}
/**
* In composite types, such as the type of the key in a map, since we use reflection to get the type class, this method
* does conversion based on the class provided.
*
- * @return converted ThriftType
+ * @param typeClass
+ * @return
+ * @throws Exception
*/
- private ThriftType convertClassToThriftType(String name, Requirement requirement, Manifest<?> typeManifest) {
- Class typeClass = typeManifest.runtimeClass();
- if (typeManifest.runtimeClass() == boolean.class) {
+ private ThriftType convertClassToThriftType(Class typeClass) {
+ if (typeClass == boolean.class) {
return new ThriftType.BoolType();
} else if (typeClass == byte.class) {
return new ThriftType.ByteType();
@@ -276,19 +273,6 @@ public class ScroogeStructConverter {
return new ThriftType.I64Type();
} else if (typeClass == String.class) {
return new ThriftType.StringType();
- } else if (typeClass == scala.collection.Seq.class) {
- Manifest<?> a = typeManifest.typeArguments().apply(0);
- return convertListTypeField(name, a, requirement);
- } else if (typeClass == scala.collection.Set.class) {
- Manifest<?> setElementManifest = typeManifest.typeArguments().apply(0);
- return convertSetTypeField(name, setElementManifest, requirement);
- } else if (typeClass == scala.collection.Map.class) {
- List<Manifest<?>> ms = JavaConversions.seqAsJavaList(typeManifest.typeArguments());
- Manifest keyManifest = ms.get(0);
- Manifest valueManifest = ms.get(1);
- return convertMapTypeField(name, keyManifest, valueManifest, requirement);
- } else if (com.twitter.scrooge.ThriftEnum.class.isAssignableFrom(typeClass)) {
- return convertEnumTypeField(typeClass, name);
} else {
return convertStructFromClass(typeClass);
}
@@ -307,65 +291,41 @@ public class ScroogeStructConverter {
Object cObject = companionObjectClass.getField("MODULE$").get(null);
Method listMethod = companionObjectClass.getMethod("list", new Class[]{});
Object result = listMethod.invoke(cObject, null);
- return JavaConversions.seqAsJavaList((Seq) result);
+ return JavaConversions.seqAsJavaList((Seq)result);
}
public ThriftType convertEnumTypeField(ThriftStructFieldInfo f) {
- return convertEnumTypeField(f.manifest().runtimeClass(), f.tfield().name);
- }
-
- private ThriftType convertEnumTypeField(Class enumClass, String fieldName) {
List<ThriftType.EnumValue> enumValues = new ArrayList<ThriftType.EnumValue>();
- String enumName = enumClass.getName();
+
+ String enumName = f.manifest().runtimeClass().getName();
try {
List enumCollection = getEnumList(enumName);
for (Object enumObj : enumCollection) {
- ScroogeEnumDesc enumDesc = ScroogeEnumDesc.fromEnum(enumObj);
+ ScroogeEnumDesc enumDesc = ScroogeEnumDesc.getEnumDesc(enumObj);
+ //be compatible with thrift generated enum which have capitalized name
enumValues.add(new ThriftType.EnumValue(enumDesc.id, enumDesc.originalName));
}
return new ThriftType.EnumType(enumValues);
} catch (ReflectiveOperationException e) {
- throw new ScroogeSchemaConversionException("Can not convert enum field " + fieldName, e);
- } catch (RuntimeException e) {
- throw new ScroogeSchemaConversionException("Can not convert enum field " + fieldName, e);
- }
-
- }
-
- //In scrooge generated class, if a class is a union, then it must have a field called "Union"
- private boolean isUnion(Class klass) {
- for (Field f : klass.getDeclaredFields()) {
- if (f.getName().equals("Union"))
- return true;
+ throw new ScroogeSchemaConversionException("Can not convert enum field " + f, e);
}
- return false;
- }
-
- private Requirement getRequirementType(ThriftStructFieldInfo f) {
- if (f.isOptional() && !f.isRequired()) {
- return OPTIONAL;
- } else if (f.isRequired() && !f.isOptional()) {
- return REQUIRED;
- } else if (!f.isOptional() && !f.isRequired()) {
- return DEFAULT;
- } else {
- throw new ScroogeSchemaConversionException("can not determine requirement type for : " + f.toString()
- + ", isOptional=" + f.isOptional() + ", isRequired=" + f.isRequired());
- }
}
private static class ScroogeEnumDesc {
private int id;
+ private String name;
private String originalName;
- public static ScroogeEnumDesc fromEnum(Object rawScroogeEnum) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ public static ScroogeEnumDesc getEnumDesc(Object rawScroogeEnum) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
Class enumClass = rawScroogeEnum.getClass();
Method valueMethod = enumClass.getMethod("value", new Class[]{});
+ Method nameMethod = enumClass.getMethod("name", new Class[]{});
Method originalNameMethod = enumClass.getMethod("originalName", new Class[]{});
ScroogeEnumDesc result = new ScroogeEnumDesc();
- result.id = (Integer) valueMethod.invoke(rawScroogeEnum, null);
- result.originalName = (String) originalNameMethod.invoke(rawScroogeEnum, null);
+ result.id = (Integer)valueMethod.invoke(rawScroogeEnum, null);
+ result.name = (String)nameMethod.invoke(rawScroogeEnum, null);
+ result.originalName = (String)originalNameMethod.invoke(rawScroogeEnum, null);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java b/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
index a20cc5b..3dc5369 100644
--- a/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
+++ b/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
@@ -18,22 +18,9 @@
*/
package parquet.scrooge;
-import org.apache.thrift.TBase;
import org.junit.Test;
-import parquet.schema.MessageType;
import parquet.scrooge.test.AddressWithStreetWithDefaultRequirement;
-import parquet.scrooge.test.ListNestEnum;
-import parquet.scrooge.test.ListNestMap;
-import parquet.scrooge.test.ListNestSet;
-import parquet.scrooge.test.MapNestList;
-import parquet.scrooge.test.MapNestMap;
-import parquet.scrooge.test.MapNestSet;
-import parquet.scrooge.test.NestedList;
-import parquet.scrooge.test.SetNestList;
-import parquet.scrooge.test.SetNestMap;
-import parquet.scrooge.test.SetNestSet;
-import parquet.scrooge.test.StringAndBinary;
import parquet.scrooge.test.TestFieldOfEnum;
import parquet.scrooge.test.TestListPrimitive;
import parquet.scrooge.test.TestMapComplex;
@@ -43,6 +30,7 @@ import parquet.scrooge.test.TestOptionalMap;
import parquet.scrooge.test.TestPersonWithAllInformation;
import parquet.scrooge.test.TestSetPrimitive;
import parquet.scrooge.test.TestUnion;
+import parquet.scrooge.test.StringAndBinary;
import parquet.thrift.ThriftSchemaConverter;
import parquet.thrift.struct.ThriftType;
import static org.junit.Assert.assertEquals;
@@ -51,128 +39,85 @@ import static org.junit.Assert.assertEquals;
* Test convert scrooge schema to Parquet Schema
*/
public class ScroogeStructConverterTest {
-
- /**
- * Convert ThriftStructs from a thrift class and a scrooge class, assert
- * they are the same
- * @param scroogeClass
- */
- private void shouldConvertConsistentlyWithThriftStructConverter(Class scroogeClass) throws ClassNotFoundException {
- Class<? extends TBase<?, ?>> thriftClass = (Class<? extends TBase<?, ?>>)Class.forName(scroogeClass.getName().replaceFirst("parquet.scrooge.test", "parquet.thrift.test"));
- ThriftType.StructType structFromThriftSchemaConverter = new ThriftSchemaConverter().toStructType(thriftClass);
- ThriftType.StructType structFromScroogeSchemaConverter = new ScroogeStructConverter().convert(scroogeClass);
-
- assertEquals(toParquetSchema(structFromThriftSchemaConverter), toParquetSchema(structFromScroogeSchemaConverter));
- }
-
- private MessageType toParquetSchema(ThriftType.StructType struct) {
- ThriftSchemaConverter sc = new ThriftSchemaConverter();
- return sc.convert(struct);
- }
-
@Test
- public void testConvertPrimitiveMapKey() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestMapPrimitiveKey.class);
+ public void testConvertPrimitiveMapKey() throws Exception{
+ ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestMapPrimitiveKey.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapPrimitiveKey.class);
+ assertEquals(expected,scroogeMap);
}
@Test
public void testBinary() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(StringAndBinary.class);
+ ThriftType.StructType scroogeBinary = new ScroogeStructConverter().convert(StringAndBinary.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.StringAndBinary.class);
+ assertEquals(expected, scroogeBinary);
}
@Test
public void testUnion() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestUnion.class);
- }
-
- @Test
- public void testConvertPrimitiveMapValue() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestMapPrimitiveValue.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestUnion.class);
+ ThriftType.StructType scroogeUnion = new ScroogeStructConverter().convert(TestUnion.class);
+ assertEquals(expected, scroogeUnion);
}
@Test
- public void testConvertPrimitiveList() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestListPrimitive.class);
+ public void testConvertPrimitiveMapValue() throws Exception{
+ ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestMapPrimitiveValue.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapPrimitiveValue.class);
+ assertEquals(expected,scroogeMap);
}
@Test
- public void testConvertPrimitiveSet() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestSetPrimitive.class);
+ public void testConvertPrimitiveList() throws Exception{
+ ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestListPrimitive.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestListPrimitive.class);
+ assertEquals(expected, scroogeList);
}
@Test
- public void testConvertEnum() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestFieldOfEnum.class);
+ public void testConvertPrimitiveSet() throws Exception{
+ ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestSetPrimitive.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestSetPrimitive.class);
+ assertEquals(expected, scroogeList);
}
@Test
- public void testMapComplex() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestMapComplex.class);
+ public void testConvertEnum() throws Exception{
+ ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestFieldOfEnum.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestFieldOfEnum.class);
+ assertEquals(expected, scroogeList);
}
@Test
- public void testConvertStruct() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestPersonWithAllInformation.class);
+ public void testMapComplex() throws Exception{
+ ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(TestMapComplex.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapComplex.class);
+ assertEquals(expected, scroogePerson);
}
@Test
- public void testDefaultFields() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(AddressWithStreetWithDefaultRequirement.class);
- }
-
- @Test
- public void testConvertOptionalPrimitiveMap() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(TestOptionalMap.class);
- }
-
- @Test
- public void testConvertNestedList() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(NestedList.class);
- }
-
- @Test
- public void testConvertListNestMap() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(ListNestMap.class);
- }
-
- @Test
- public void testConvertListNestEnum() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(ListNestEnum.class);
- }
-
- @Test
- public void testConvertMapNestList() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(MapNestList.class);
- }
-
- @Test
- public void testConvertMapNestMap() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(MapNestMap.class);
- }
-
- @Test
- public void testConvertMapNestSet() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(MapNestSet.class);
- }
-
- @Test
- public void testConvertListNestSet() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(ListNestSet.class);
- }
-
- @Test
- public void testConvertSetNestSet() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(SetNestSet.class);
+ public void testConvertStruct() throws Exception{
+ ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(TestPersonWithAllInformation.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestPersonWithAllInformation.class);
+ assertEquals(expected, scroogePerson);
}
+/**
+ * TODO: DEFAULT requirement can not be identified, since scrooge does not store the requirement type in generated class
+ * Current solution uses reflection based on following rules:
+ * if the getter returns option, then it's optional, otherwise it's required
+ */
@Test
- public void testConvertSetNestList() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(SetNestList.class);
+ public void testDefaultFields() throws Exception{
+ ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(AddressWithStreetWithDefaultRequirement.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.AddressWithStreetWithDefaultRequirement.class);
+ assertEquals(expected.toJSON(), scroogePerson.toJSON());
}
@Test
- public void testConvertSetNestMap() throws Exception {
- shouldConvertConsistentlyWithThriftStructConverter(SetNestMap.class);
+ public void testConvertOptionalPrimitiveMap() throws Exception{
+ ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestOptionalMap.class);
+ ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestOptionalMap.class);
+ assertEquals(expected,scroogeMap);
}
-
}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/test/thrift/test.thrift
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/thrift/test.thrift b/parquet-scrooge/src/test/thrift/test.thrift
index f721193..a80bbb0 100644
--- a/parquet-scrooge/src/test/thrift/test.thrift
+++ b/parquet-scrooge/src/test/thrift/test.thrift
@@ -169,95 +169,6 @@ struct TestFieldOfEnum{
}
struct StringAndBinary {
- 1: required string s
- 2: required binary b
-}
-
-#fixture fox nested structures
-struct NestedList {
- 1: required list<list<Address>> rll
- 2: required list<list<list<Address>>> rlll
- 3: optional list<list<Address>> oll
- 4: optional list<list<list<Address>>> olll
- 5: list<list<Address>> ll
- 6: list<list<list<Address>>> lll
-}
-
-struct ListNestMap {
- 1: required list<map<Phone, Address>> rlm
- 2: required list<list<map<Phone, Address>>> rllm
- 3: optional list<map<Phone, Address>> olm
- 4: optional list<list<map<Phone, Address>>> ollm
- 5: list<map<Phone, Address>> lm
- 6: list<list<map<Phone, Address>>> llm
-}
-
-struct ListNestSet {
- 1: required list<set<Address>> rls
- 2: required list<list<set<Address>>> rlls
- 3: optional list<set<Address>> ols
- 4: optional list<list<set<Address>>> olls
- 5: list<set<Address>> ls
- 6: list<list<set<Address>>> lls
-}
-
-struct ListNestEnum {
- 1: required list<Operation> rle
-}
-
-struct MapNestMap {
- 1: required map<map<Phone, Address>, map<Address, Phone>> rmm
- 2: required map<map<map<Phone,Address>, Address>, map<Address, Phone>> rmmm
- 3: optional map<map<Phone, Address>, map<Address, Phone>> omm
- 4: optional map<map<map<Phone,Address>, Address>, map<Address, Phone>> ommm
- 5: map<map<Phone, Address>, map<Address, Phone>> mm
- 6: map<map<map<Phone,Address>, Address>, map<Address, Phone>> mmm
-}
-
-struct MapNestList {
- 1: required map<list<Phone>, list<Address>> rml
- 2: required map<list<list<Phone>>, list<list<Address>>> rmll
- 3: optional map<list<Phone>, list<Address>> oml
- 4: optional map<list<list<Phone>>, list<list<Address>>> omll
- 5: map<list<Phone>, list<Address>> ml
- 6: map<list<list<Phone>>, list<list<Address>>> mll
-}
-
-struct MapNestSet {
- 1: required map<set<Phone>, set<Address>> rms
- 2: required map<set<set<Phone>>, set<set<Address>>> rmss
- 3: optional map<set<Phone>, set<Address>> oms
- 4: optional map<set<set<Phone>>, set<set<Address>>> omss
- 5: map<set<Phone>, set<Address>> ms
- 6: map<set<set<Phone>>, set<set<Address>>> mss
-}
-
-struct SetNestSet {
- 1: required set<set<Address>> rss
- 2: required set<set<set<Address>>> rsss
- 3: optional set<set<Address>> oss
- 4: optional set<set<set<Address>>> osss
- 5: set<set<Address>> ss
- 6: set<set<set<Address>>> sss
-}
-
-struct SetNestList {
- 1: required set<list<Address>> rsl
- 2: required set<set<list<Address>>> rssl
- 3: optional set<list<Address>> osl
- 4: optional set<set<list<Address>>> ossl
- 5: set<list<Address>> sl
- 6: set<set<list<Address>>> ssl
-}
-
-struct SetNestMap {
- 1: required set<map<Phone, Address>> rsm
- 2: required set<set<map<Phone, Address>>> rssm
- 3: required set<set<list<list<map<Phone, Address>>>>> rssllm
- 4: optional set<map<Phone, Address>> osm
- 5: optional set<set<map<Phone, Address>>> ossm
- 6: optional set<set<list<list<map<Phone, Address>>>>> ossllm
- 7: set<map<Phone, Address>> sm
- 8: set<set<map<Phone, Address>>> ssm
- 9: set<set<list<list<map<Phone, Address>>>>> ssllm
+ 1: required string s;
+ 2: required binary b;
}