You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/08/19 07:11:30 UTC
svn commit: r1618794 - in /hive/trunk:
ql/src/java/org/apache/hadoop/hive/ql/io/avro/
serde/src/java/org/apache/hadoop/hive/serde2/avro/
serde/src/test/org/apache/hadoop/hive/serde2/avro/
Author: brock
Date: Tue Aug 19 05:11:30 2014
New Revision: 1618794
URL: http://svn.apache.org/r1618794
Log:
HIVE-7432 - Remove deprecated Avro's Schema.parse usages (Ashish Kumar Singh via Brock)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java Tue Aug 19 05:11:30 2014
@@ -127,7 +127,7 @@ public class AvroGenericRecordReader imp
String s = job.get(AvroSerdeUtils.AVRO_SERDE_SCHEMA);
if(s != null) {
LOG.info("Found the avro schema in the job: " + s);
- return Schema.parse(s);
+ return AvroSerdeUtils.getSchemaFor(s);
}
// No more places to get the schema from. Give up. May have to re-encode later.
return null;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java Tue Aug 19 05:11:30 2014
@@ -93,8 +93,8 @@ public class AvroGenericRecordWritable i
@Override
public void readFields(DataInput in) throws IOException {
- Schema schema = Schema.parse(in.readUTF());
- fileSchema = Schema.parse(in.readUTF());
+ Schema schema = AvroSerdeUtils.getSchemaFor(in.readUTF());
+ fileSchema = AvroSerdeUtils.getSchemaFor(in.readUTF());
recordReaderID = UID.read(in);
record = new GenericData.Record(schema);
binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder((InputStream) in, binaryDecoder);
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java Tue Aug 19 05:11:30 2014
@@ -29,7 +29,9 @@ import org.apache.hadoop.hive.common.typ
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.mapred.JobConf;
+import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
@@ -67,7 +69,7 @@ public class AvroSerdeUtils {
throws IOException, AvroSerdeException {
String schemaString = properties.getProperty(SCHEMA_LITERAL);
if(schemaString != null && !schemaString.equals(SCHEMA_NONE))
- return Schema.parse(schemaString);
+ return AvroSerdeUtils.getSchemaFor(schemaString);
// Try pulling directly from URL
schemaString = properties.getProperty(SCHEMA_URL);
@@ -78,7 +80,7 @@ public class AvroSerdeUtils {
Schema s = getSchemaFromFS(schemaString, new Configuration());
if (s == null) {
//in case schema is not a file system
- return Schema.parse(new URL(schemaString).openStream());
+ return AvroSerdeUtils.getSchemaFor(new URL(schemaString).openStream());
}
return s;
} catch (IOException ioe) {
@@ -123,7 +125,7 @@ public class AvroSerdeUtils {
}
try {
in = fs.open(new Path(schemaFSUrl));
- Schema s = Schema.parse(in);
+ Schema s = AvroSerdeUtils.getSchemaFor(in);
return s;
} finally {
if(in != null) in.close();
@@ -194,4 +196,31 @@ public class AvroSerdeUtils {
return dec;
}
+ public static Schema getSchemaFor(String str) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema = parser.parse(str);
+ return schema;
+ }
+
+ public static Schema getSchemaFor(File file) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema;
+ try {
+ schema = parser.parse(file);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse Avro schema from " + file.getName(), e);
+ }
+ return schema;
+ }
+
+ public static Schema getSchemaFor(InputStream stream) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema;
+ try {
+ schema = parser.parse(stream);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse Avro schema", e);
+ }
+ return schema;
+ }
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java Tue Aug 19 05:11:30 2014
@@ -55,5 +55,5 @@ class SchemaResolutionProblem {
" }\n" +
" ]\n" +
"}";
- public final static Schema SIGNAL_BAD_SCHEMA = Schema.parse(sentinelString);
+ public final static Schema SIGNAL_BAD_SCHEMA = AvroSerdeUtils.getSchemaFor(sentinelString);
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java Tue Aug 19 05:11:30 2014
@@ -38,7 +38,6 @@ import java.util.List;
*/
public class TypeInfoToSchema {
- private static final Schema.Parser PARSER = new Schema.Parser();
private long recordCounter = 0;
/**
@@ -139,7 +138,7 @@ public class TypeInfoToSchema {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
String precision = String.valueOf(decimalTypeInfo.precision());
String scale = String.valueOf(decimalTypeInfo.scale());
- schema = PARSER.parse("{" +
+ schema = AvroSerdeUtils.getSchemaFor("{" +
"\"type\":\"bytes\"," +
"\"logicalType\":\"decimal\"," +
"\"precision\":" + precision + "," +
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java Tue Aug 19 05:11:30 2014
@@ -55,7 +55,7 @@ public class TestAvroDeserializer {
" {\"name\": \"isANull\", \"type\": \"null\"}\n" +
" ]\n" +
"}";
- Schema s = Schema.parse(schemaString);
+ Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
GenericData.Record record = new GenericData.Record(s);
record.put("isANull", null);
@@ -83,7 +83,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeMapsWithPrimitiveKeys() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.MAP_WITH_PRIMITIVE_VALUE_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.MAP_WITH_PRIMITIVE_VALUE_TYPE);
GenericData.Record record = new GenericData.Record(s);
Map<String, Long> m = new Hashtable<String, Long>();
@@ -129,7 +129,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeArrays() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
GenericData.Record record = new GenericData.Record(s);
List<String> list = new ArrayList<String>();
@@ -187,7 +187,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeRecords() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
innerRecord.put("int1", 42);
@@ -246,7 +246,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeUnions() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.UNION_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.UNION_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
record.put("aUnion", "this is a string");
@@ -295,7 +295,7 @@ public class TestAvroDeserializer {
@Test // Enums are one of two types we fudge for Hive. Enums go in, Strings come out.
public void canDeserializeEnums() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.ENUM_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
record.put("baddies", new GenericData.EnumSymbol(s.getField("baddies").schema(),"DALEKS"));
@@ -325,7 +325,7 @@ public class TestAvroDeserializer {
@Test // Fixed doesn't exist in Hive. Fixeds go in, lists of bytes go out.
public void canDeserializeFixed() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.FIXED_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.FIXED_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
byte [] bytes = "ANANCIENTBLUEBOX".getBytes();
@@ -361,7 +361,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeBytes() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.BYTES_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.BYTES_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
byte [] bytes = "ANANCIENTBLUEBOX".getBytes();
@@ -400,7 +400,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeNullableTypes() throws IOException, SerDeException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.NULLABLE_STRING_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.NULLABLE_STRING_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
record.put("nullableString", "this is a string");
@@ -413,7 +413,7 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeNullableEnums() throws IOException, SerDeException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.NULLABLE_ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.NULLABLE_ENUM_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
record.put("nullableEnum", new GenericData.EnumSymbol(AvroSerdeUtils.getOtherTypeFromNullableType(s.getField("nullableEnum").schema()), "CYBERMEN"));
@@ -426,7 +426,8 @@ public class TestAvroDeserializer {
@Test
public void canDeserializeMapWithNullablePrimitiveValues() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator
+ .MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
Map<String, Long> m = new HashMap<String, Long>();
@@ -504,7 +505,7 @@ public class TestAvroDeserializer {
@Test
public void verifyCaching() throws SerDeException, IOException {
- Schema s = Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
innerRecord.put("int1", 42);
@@ -541,7 +542,7 @@ public class TestAvroDeserializer {
assertEquals(0, de.getReEncoderCache().size());
//Read the record with **different** record reader ID and **evolved** schema
- Schema evolvedSchema = Schema.parse(s.toString());
+ Schema evolvedSchema = AvroSerdeUtils.getSchemaFor(s.toString());
evolvedSchema.getField("aRecord").schema().addProp("Testing", "meaningless");
garw.setRecordReaderID(recordReaderID = new UID()); //New record reader ID
row =
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java Tue Aug 19 05:11:30 2014
@@ -41,13 +41,13 @@ import org.apache.hadoop.hive.serde2.typ
import org.junit.Test;
public class TestAvroObjectInspectorGenerator {
- private final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string");
- private final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int");
- private final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean");
- private final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
- private final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float");
- private final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double");
- private final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void");
+ private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string");
+ private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int");
+ private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean");
+ private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
+ private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float");
+ private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double");
+ private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void");
// These schemata are used in other tests
static public final String MAP_WITH_PRIMITIVE_VALUE_TYPE = "{\n" +
@@ -265,7 +265,7 @@ public class TestAvroObjectInspectorGene
" \"symbols\" : [\"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\"]\n" +
"}";
- Schema s = Schema.parse(nonRecordSchema);
+ Schema s = AvroSerdeUtils.getSchemaFor(nonRecordSchema);
try {
new AvroObjectInspectorGenerator(s);
fail("Should not be able to handle non-record Avro types");
@@ -311,7 +311,7 @@ public class TestAvroObjectInspectorGene
" }\n" +
" ]\n" +
"}";
- AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(Schema.parse(bunchOfPrimitives));
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(AvroSerdeUtils.getSchemaFor(bunchOfPrimitives));
String [] expectedColumnNames = {"aString", "anInt", "aBoolean", "aLong", "aFloat", "aDouble", "aNull"};
verifyColumnNames(expectedColumnNames, aoig.getColumnNames());
@@ -350,7 +350,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException {
- Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(MAP_WITH_PRIMITIVE_VALUE_TYPE);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
verifyMap(aoig, "aMap");
}
@@ -379,7 +379,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleArrays() throws SerDeException {
- Schema s = Schema.parse(ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -398,7 +398,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleRecords() throws SerDeException {
- Schema s = Schema.parse(RECORD_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(RECORD_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -429,7 +429,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleUnions() throws SerDeException {
- Schema s = Schema.parse(UNION_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(UNION_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -452,7 +452,7 @@ public class TestAvroObjectInspectorGene
@Test // Enums are one of two Avro types that Hive doesn't have any native support for.
public void canHandleEnums() throws SerDeException {
- Schema s = Schema.parse(ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(ENUM_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names - we lose the enumness of this schema
@@ -466,7 +466,7 @@ public class TestAvroObjectInspectorGene
@Test // Hive has no concept of Avro's fixed type. Fixed -> arrays of bytes
public void canHandleFixed() throws SerDeException {
- Schema s = Schema.parse(FIXED_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(FIXED_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
@@ -483,7 +483,7 @@ public class TestAvroObjectInspectorGene
@Test // Avro considers bytes primitive, Hive doesn't. Make them list of tinyint.
public void canHandleBytes() throws SerDeException {
- Schema s = Schema.parse(BYTES_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(BYTES_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
@@ -500,7 +500,7 @@ public class TestAvroObjectInspectorGene
@Test // That Union[T, NULL] is converted to just T.
public void convertsNullableTypes() throws SerDeException {
- Schema s = Schema.parse(NULLABLE_STRING_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_STRING_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
assertEquals(1, aoig.getColumnNames().size());
@@ -517,14 +517,14 @@ public class TestAvroObjectInspectorGene
@Test // That Union[T, NULL] is converted to just T, within a Map
public void convertsMapsWithNullablePrimitiveTypes() throws SerDeException {
- Schema s = Schema.parse(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
verifyMap(aoig, "aMap");
}
@Test // That Union[T, NULL] is converted to just T.
public void convertsNullableEnum() throws SerDeException {
- Schema s = Schema.parse(NULLABLE_ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_ENUM_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
assertEquals(1, aoig.getColumnNames().size());
@@ -542,10 +542,10 @@ public class TestAvroObjectInspectorGene
@Test
public void objectInspectorsAreCached() throws SerDeException {
// Verify that Hive is caching the object inspectors for us.
- Schema s = Schema.parse(KITCHEN_SINK_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(KITCHEN_SINK_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
- Schema s2 = Schema.parse(KITCHEN_SINK_SCHEMA);
+ Schema s2 = AvroSerdeUtils.getSchemaFor(KITCHEN_SINK_SCHEMA);
AvroObjectInspectorGenerator aoig2 = new AvroObjectInspectorGenerator(s2);
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java Tue Aug 19 05:11:30 2014
@@ -61,8 +61,8 @@ public class TestAvroSerde {
" ]\n" +
"}";
- static final Schema originalSchema = Schema.parse(originalSchemaString);
- static final Schema newSchema = Schema.parse(newSchemaString);
+ static final Schema originalSchema = AvroSerdeUtils.getSchemaFor(originalSchemaString);
+ static final Schema newSchema = AvroSerdeUtils.getSchemaFor(newSchemaString);
@Test
public void initializeDoesNotReuseSchemasFromConf() throws SerDeException {
@@ -81,7 +81,7 @@ public class TestAvroSerde {
// Verify that the schema now within the configuration is the one passed
// in via the properties
- assertEquals(newSchema, Schema.parse(conf.get(AVRO_SERDE_SCHEMA)));
+ assertEquals(newSchema, AvroSerdeUtils.getSchemaFor(conf.get(AVRO_SERDE_SCHEMA)));
}
@Test
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java Tue Aug 19 05:11:30 2014
@@ -58,7 +58,7 @@ public class TestAvroSerdeUtils {
"}";
private void testField(String schemaString, String fieldName, boolean shouldBeNullable) {
- Schema s = Schema.parse(schemaString);
+ Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
assertEquals(shouldBeNullable, isNullableType(s.getField(fieldName).schema()));
}
@@ -106,11 +106,11 @@ public class TestAvroSerdeUtils {
@Test
public void getTypeFromNullableTypePositiveCase() {
- Schema s = Schema.parse(NULLABLE_UNION);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_UNION);
Schema typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
- s = Schema.parse(NULLABLE_UNION2);
+ s = AvroSerdeUtils.getSchemaFor(NULLABLE_UNION2);
typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
}
@@ -126,7 +126,7 @@ public class TestAvroSerdeUtils {
String schema = TestAvroObjectInspectorGenerator.RECORD_SCHEMA;
Properties props = new Properties();
props.put(AvroSerdeUtils.SCHEMA_LITERAL, schema);
- Schema expected = Schema.parse(schema);
+ Schema expected = AvroSerdeUtils.getSchemaFor(schema);
assertEquals(expected, AvroSerdeUtils.determineSchemaOrThrowException(props));
}
@@ -163,7 +163,7 @@ public class TestAvroSerdeUtils {
try {
s = determineSchemaOrThrowException(props);
assertNotNull(s);
- assertEquals(Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s);
+ assertEquals(AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s);
} catch(AvroSerdeException he) {
fail("Should have parsed schema literal, not thrown exception.");
}
@@ -197,7 +197,7 @@ public class TestAvroSerdeUtils {
Schema schemaFromHDFS =
AvroSerdeUtils.getSchemaFromFS(onHDFS, miniDfs.getFileSystem().getConf());
- Schema expectedSchema = Schema.parse(schemaString);
+ Schema expectedSchema = AvroSerdeUtils.getSchemaFor(schemaString);
assertEquals(expectedSchema, schemaFromHDFS);
} finally {
if(miniDfs != null) miniDfs.shutdown();
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java Tue Aug 19 05:11:30 2014
@@ -53,7 +53,7 @@ public class TestAvroSerializer {
" \"fields\": [" +
recordValues +
" ] }";
- return Schema.parse(s);
+ return AvroSerdeUtils.getSchemaFor(s);
}
/**
@@ -231,7 +231,7 @@ public class TestAvroSerializer {
public void canSerializeEnums() throws SerDeException, IOException {
String type = "{\"type\": \"enum\", \"name\": \"enum1_values\", " +
"\"symbols\":[\"BLUE\",\"RED\",\"GREEN\"]}";
- Schema schema = Schema.parse(type);
+ Schema schema = AvroSerdeUtils.getSchemaFor(type);
String field = "{ \"name\":\"enum1\", \"type\": " + schema + " }";
for(enum1 e : enum1.values()) {
GenericEnumSymbol symbol = new GenericData.EnumSymbol(schema, e.toString());
@@ -247,7 +247,7 @@ public class TestAvroSerializer {
String type = "{\"type\": \"enum\", \"name\": \"enum1_values\",\n" +
" \"namespace\": \"org.apache.hadoop.hive\",\n" +
" \"symbols\":[\"BLUE\",\"RED\",\"GREEN\"]}";
- Schema schema = Schema.parse(type);
+ Schema schema = AvroSerdeUtils.getSchemaFor(type);
String field = "{ \"name\":\"nullableenum\", \"type\": [\"null\", " + schema + "] }";
GenericEnumSymbol symbol = new GenericData.EnumSymbol(schema, enum1.BLUE.toString());
GenericRecord r = serializeAndDeserialize(field, "nullableenum", symbol);
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java Tue Aug 19 05:11:30 2014
@@ -50,7 +50,7 @@ public class TestGenericAvroRecordWritab
@Test
public void writableContractIsImplementedCorrectly() throws IOException {
- Schema schema = Schema.parse(schemaJSON);
+ Schema schema = AvroSerdeUtils.getSchemaFor(schemaJSON);
GenericRecord gr = new GenericData.Record(schema);
gr.put("first", "The");
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java Tue Aug 19 05:11:30 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.Ser
import org.junit.Test;
public class TestSchemaReEncoder {
+
@Test
public void schemasCanAddFields() throws SerDeException {
String original = "{\n" +
@@ -56,8 +57,8 @@ public class TestSchemaReEncoder {
" }\n" +
" ]\n" +
"}";
- Schema originalSchema = Schema.parse(original);
- Schema evolvedSchema = Schema.parse(evolved);
+ Schema originalSchema = AvroSerdeUtils.getSchemaFor(original);
+ Schema evolvedSchema = AvroSerdeUtils.getSchemaFor(evolved);
GenericRecord record = new GenericData.Record(originalSchema);
record.put("text", "it is a far better thing I do, yadda, yadda");
@@ -97,8 +98,8 @@ public class TestSchemaReEncoder {
" }\n" +
" ]\n" +
"}";
- Schema originalSchema2 = Schema.parse(original2);
- Schema evolvedSchema2 = Schema.parse(evolved2);
+ Schema originalSchema2 = AvroSerdeUtils.getSchemaFor(original2);
+ Schema evolvedSchema2 = AvroSerdeUtils.getSchemaFor(evolved2);
record = new GenericData.Record(originalSchema2);
record.put("a", 19);
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java?rev=1618794&r1=1618793&r2=1618794&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java Tue Aug 19 05:11:30 2014
@@ -34,6 +34,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
public class TestThatEvolvedSchemasActAsWeWant {
+
@Test
public void resolvedSchemasShouldReturnReaderSchema() throws IOException {
// Need to verify that when reading a datum with an updated reader schema
@@ -68,7 +69,7 @@ public class TestThatEvolvedSchemasActAs
" ]\n" +
"}";
- Schema[] schemas = {Schema.parse(v0), Schema.parse(v1)};
+ Schema[] schemas = {AvroSerdeUtils.getSchemaFor(v0), AvroSerdeUtils.getSchemaFor(v1)};
// Encode a schema with v0, write out.
GenericRecord record = new GenericData.Record(schemas[0]);