You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ec...@apache.org on 2012/06/02 04:14:06 UTC
svn commit: r1345420 [3/3] - in /hive/trunk: data/files/
ql/src/java/org/apache/hadoop/hive/ql/io/avro/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/ serde/src/java/org/apache/hadoop/hive/serde2/avro/
serde/src/test/org...
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.Properties;
+
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.EXCEPTION_MESSAGE;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.SCHEMA_LITERAL;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.SCHEMA_NONE;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.SCHEMA_URL;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.determineSchemaOrThrowException;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.getOtherTypeFromNullableType;
+import static org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.isNullableType;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestAvroSerdeUtils {
+ private final String NULLABLE_UNION = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"nullTest\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"mayBeNull\", \"type\":[\"string\", \"null\"]}\n" +
+ " ]\n" +
+ "}";
+ // Same union, order reveresed
+ private final String NULLABLE_UNION2 = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"nullTest\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"mayBeNull\", \"type\":[\"null\", \"string\"]}\n" +
+ " ]\n" +
+ "}";
+
+ private void testField(String schemaString, String fieldName, boolean shouldBeNullable) {
+ Schema s = Schema.parse(schemaString);
+ assertEquals(shouldBeNullable, isNullableType(s.getField(fieldName).schema()));
+ }
+
+ @Test
+ public void isNullableTypeAcceptsNullableUnions() {
+ testField(NULLABLE_UNION, "mayBeNull", true);
+ testField(NULLABLE_UNION2, "mayBeNull", true);
+ }
+
+ @Test
+ public void isNullableTypeIdentifiesUnionsOfMoreThanTwoTypes() {
+ String schemaString = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"shouldNotPass\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"mayBeNull\", \"type\":[\"string\", \"int\", \"null\"]}\n" +
+ " ]\n" +
+ "}";
+ testField(schemaString, "mayBeNull", false);
+ }
+
+ @Test
+ public void isNullableTypeIdentifiesUnionsWithoutNulls() {
+ String s = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"unionButNoNull\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"a\", \"type\":[\"int\", \"string\"]}\n" +
+ " ]\n" +
+ "}";
+ testField(s, "a", false);
+ }
+
+ @Test
+ public void isNullableTypeIdentifiesNonUnionTypes() {
+ String schemaString = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"nullTest2\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"justAnInt\", \"type\":\"int\"}\n" +
+ " ]\n" +
+ "}";
+ testField(schemaString, "justAnInt", false);
+ }
+
+ @Test
+ public void getTypeFromNullableTypePositiveCase() {
+ Schema s = Schema.parse(NULLABLE_UNION);
+ Schema typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
+ assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
+
+ s = Schema.parse(NULLABLE_UNION2);
+ typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
+ assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
+ }
+
+ @Test(expected=AvroSerdeException.class)
+ public void determineSchemaThrowsExceptionIfNoSchema() throws IOException, AvroSerdeException {
+ Properties prop = new Properties();
+ AvroSerdeUtils.determineSchemaOrThrowException(prop);
+ }
+
+ @Test
+ public void determineSchemaFindsLiterals() throws Exception {
+ String schema = TestAvroObjectInspectorGenerator.RECORD_SCHEMA;
+ Properties props = new Properties();
+ props.put(AvroSerdeUtils.SCHEMA_LITERAL, schema);
+ Schema expected = Schema.parse(schema);
+ assertEquals(expected, AvroSerdeUtils.determineSchemaOrThrowException(props));
+ }
+
+ @Test
+ public void detemineSchemaTriesToOpenUrl() throws AvroSerdeException, IOException {
+ Properties props = new Properties();
+ props.put(AvroSerdeUtils.SCHEMA_URL, "not:///a.real.url");
+
+ try {
+ AvroSerdeUtils.determineSchemaOrThrowException(props);
+ fail("Should have tried to open that URL");
+ } catch(MalformedURLException e) {
+ assertEquals("unknown protocol: not", e.getMessage());
+ }
+ }
+
+ @Test
+ public void noneOptionWorksForSpecifyingSchemas() throws IOException, AvroSerdeException {
+ Properties props = new Properties();
+
+ // Combo 1: Both set to none
+ props.put(SCHEMA_URL, SCHEMA_NONE);
+ props.put(SCHEMA_LITERAL, SCHEMA_NONE);
+ try {
+ determineSchemaOrThrowException(props);
+ fail("Should have thrown exception with none set for both url and literal");
+ } catch(AvroSerdeException he) {
+ assertEquals(EXCEPTION_MESSAGE, he.getMessage());
+ }
+
+ // Combo 2: Literal set, url set to none
+ props.put(SCHEMA_LITERAL, TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+ Schema s;
+ try {
+ s = determineSchemaOrThrowException(props);
+ assertNotNull(s);
+ assertEquals(Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s);
+ } catch(AvroSerdeException he) {
+ fail("Should have parsed schema literal, not thrown exception.");
+ }
+
+ // Combo 3: url set, literal set to none
+ props.put(SCHEMA_LITERAL, SCHEMA_NONE);
+ props.put(SCHEMA_URL, "not:///a.real.url");
+ try {
+ determineSchemaOrThrowException(props);
+ fail("Should have tried to open that bogus URL");
+ } catch(MalformedURLException e) {
+ assertEquals("unknown protocol: not", e.getMessage());
+ }
+ }
+
+ @Test
+ public void determineSchemaCanReadSchemaFromHDFS() throws IOException, AvroSerdeException {
+ String schemaString = TestAvroObjectInspectorGenerator.RECORD_SCHEMA;
+ MiniDFSCluster miniDfs = null;
+ try {
+ // MiniDFSCluster litters files and folders all over the place.
+ miniDfs = new MiniDFSCluster(new Configuration(), 1, true, null);
+
+ miniDfs.getFileSystem().mkdirs(new Path("/path/to/schema"));
+ FSDataOutputStream out = miniDfs.getFileSystem()
+ .create(new Path("/path/to/schema/schema.avsc"));
+ out.writeBytes(schemaString);
+ out.close();
+ String onHDFS = miniDfs.getFileSystem().getUri() + "/path/to/schema/schema.avsc";
+
+ Schema schemaFromHDFS =
+ AvroSerdeUtils.getSchemaFromHDFS(onHDFS, miniDfs.getFileSystem().getConf());
+ Schema expectedSchema = Schema.parse(schemaString);
+ assertEquals(expectedSchema, schemaFromHDFS);
+ } finally {
+ if(miniDfs != null) miniDfs.shutdown();
+ }
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Writable;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TestAvroSerializer {
+
+ private Schema buildSchema(String recordValues) {
+ String s = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"test_serializer\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [" +
+ recordValues +
+ " ] }";
+ return Schema.parse(s);
+ }
+
+ /**
+ * Verify that we can serialize an avro value by taking one, running it through
+ * the deser process and then serialize it again.
+ */
+ private GenericRecord serializeAndDeserialize(String recordValue,
+ String fieldName, Object fieldValue) throws SerDeException, IOException {
+ Schema s = buildSchema(recordValue);
+ GenericData.Record r = new GenericData.Record(s);
+ r.put(fieldName, fieldValue);
+
+ AvroSerializer as = new AvroSerializer();
+
+ AvroDeserializer ad = new AvroDeserializer();
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+ ObjectInspector oi = aoig.getObjectInspector();
+ List<String> columnNames = aoig.getColumnNames();
+ List<TypeInfo> columnTypes = aoig.getColumnTypes();
+
+ AvroGenericRecordWritable agrw = Utils.serializeAndDeserializeRecord(r);
+ Object obj = ad.deserialize(columnNames, columnTypes, agrw, s);
+
+ Writable result = as.serialize(obj, oi, columnNames, columnTypes, s);
+ assertTrue(result instanceof AvroGenericRecordWritable);
+ GenericRecord r2 = ((AvroGenericRecordWritable) result).getRecord();
+ assertEquals(s, r2.getSchema());
+ return r2;
+ }
+
+ @Test
+ public void canSerializeStrings() throws SerDeException, IOException {
+ singleFieldTest("string1", "hello", "string");
+ }
+
+ private void singleFieldTest(String fieldName, Object fieldValue, String fieldType)
+ throws SerDeException, IOException {
+ GenericRecord r2 = serializeAndDeserialize("{ \"name\":\"" + fieldName +
+ "\", \"type\":\"" + fieldType + "\" }", fieldName, fieldValue);
+ assertEquals(fieldValue, r2.get(fieldName));
+ }
+
+ @Test
+ public void canSerializeInts() throws SerDeException, IOException {
+ singleFieldTest("int1", 42, "int");
+ }
+
+ @Test
+ public void canSerializeBooleans() throws SerDeException, IOException {
+ singleFieldTest("boolean1", true, "boolean");
+ }
+
+ @Test
+ public void canSerializeFloats() throws SerDeException, IOException {
+ singleFieldTest("float1", 42.24342f, "float");
+ }
+
+ @Test
+ public void canSerializeDoubles() throws SerDeException, IOException {
+ singleFieldTest("double1", 24.00000001, "double");
+ }
+
+ @Test
+ public void canSerializeLists() throws SerDeException, IOException {
+ List<Integer> intList = new ArrayList<Integer>();
+ Collections.addAll(intList, 1,2, 3);
+ String field = "{ \"name\":\"list1\", \"type\":{\"type\":\"array\", \"items\":\"int\"} }";
+ GenericRecord r = serializeAndDeserialize(field, "list1", intList);
+ assertEquals(intList, r.get("list1"));
+ }
+
+ @Test
+ public void canSerializeMaps() throws SerDeException, IOException {
+ Map<String, Boolean> m = new Hashtable<String, Boolean>();
+ m.put("yes", true);
+ m.put("no", false);
+ String field = "{ \"name\":\"map1\", \"type\":{\"type\":\"map\", \"values\":\"boolean\"} }";
+ GenericRecord r = serializeAndDeserialize(field, "map1", m);
+
+ assertEquals(m, r.get("map1"));
+ }
+
+ @Test
+ public void canSerializeStructs() throws SerDeException {
+ String field = "{ \"name\":\"struct1\", \"type\":{\"type\":\"record\", " +
+ "\"name\":\"struct1_name\", \"fields\": [\n" +
+ "{ \"name\":\"sInt\", \"type\":\"int\" }, { \"name\"" +
+ ":\"sBoolean\", \"type\":\"boolean\" }, { \"name\":\"sString\", \"type\":\"string\" } ] } }";
+
+ Schema s = buildSchema(field);
+ GenericData.Record innerRecord = new GenericData.Record(s.getField("struct1").schema());
+
+ innerRecord.put("sInt", 77);
+ innerRecord.put("sBoolean", false);
+ innerRecord.put("sString", "tedious");
+
+ GenericData.Record r = new GenericData.Record(s);
+ r.put("struct1", innerRecord);
+
+ AvroSerializer as = new AvroSerializer();
+
+ AvroDeserializer ad = new AvroDeserializer();
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+ ObjectInspector oi = aoig.getObjectInspector();
+ List<String> columnNames = aoig.getColumnNames();
+ List<TypeInfo> columnTypes = aoig.getColumnTypes();
+ AvroGenericRecordWritable agrw = new AvroGenericRecordWritable(r);
+ Object obj = ad.deserialize(columnNames, columnTypes, agrw, s);
+
+ Writable result = as.serialize(obj, oi, columnNames, columnTypes, s);
+ assertTrue(result instanceof AvroGenericRecordWritable);
+ GenericRecord r2 = ((AvroGenericRecordWritable) result).getRecord();
+ assertEquals(s, r2.getSchema());
+
+ GenericRecord r3 = (GenericRecord)r2.get("struct1");
+ assertEquals(77, r3.get("sInt"));
+ assertEquals(false, r3.get("sBoolean"));
+ assertEquals("tedious", r3.get("sString"));
+ }
+
+ @Test
+ public void canSerializeUnions() throws SerDeException, IOException {
+ String field = "{ \"name\":\"union1\", \"type\":[\"float\", \"boolean\", \"string\"] }";
+ GenericRecord r = serializeAndDeserialize(field, "union1", 424.4f);
+ assertEquals(424.4f, r.get("union1"));
+
+ r = serializeAndDeserialize(field, "union1", true);
+ assertEquals(true, r.get("union1"));
+
+ r = serializeAndDeserialize(field, "union1", "hello");
+ assertEquals("hello", r.get("union1"));
+ }
+
+ private enum enum1 {BLUE, RED , GREEN};
+ @Test
+ public void canSerializeEnums() throws SerDeException, IOException {
+ for(enum1 e : enum1.values()) {
+ String field = "{ \"name\":\"enum1\", \"type\":{\"type\":\"enum\", " +
+ "\"name\":\"enum1_values\", \"symbols\":[\"BLUE\",\"RED\", \"GREEN\"]} }";
+ GenericRecord r = serializeAndDeserialize(field, "enum1", e);
+
+ assertEquals(e, enum1.valueOf(r.get("enum1").toString()));
+ }
+
+ }
+
+ @Test
+ public void canSerializeNullableTypes() throws SerDeException, IOException {
+ String field = "{ \"name\":\"nullableint\", \"type\":[\"int\", \"null\"] }";
+ GenericRecord r = serializeAndDeserialize(field, "nullableint", 42);
+ assertEquals(42, r.get("nullableint"));
+
+ r = serializeAndDeserialize(field, "nullableint", null);
+ assertNull(r.get("nullableint"));
+ }
+
+ @Test
+ public void canSerializeBytes() throws SerDeException, IOException {
+ String field = "{ \"name\":\"bytes1\", \"type\":\"bytes\" }";
+ ByteBuffer bb = ByteBuffer.wrap("easy as one two three".getBytes());
+ bb.rewind();
+ GenericRecord r = serializeAndDeserialize(field, "bytes1", bb);
+
+ assertEquals(bb, r.get("bytes1"));
+ }
+
+ @Test
+ public void canSerializeFixed() throws SerDeException, IOException {
+ String field = "{ \"name\":\"fixed1\", \"type\":{\"type\":\"fixed\", " +
+ "\"name\":\"threebytes\", \"size\":3} }";
+ GenericData.Fixed fixed = new GenericData.Fixed(buildSchema(field), "k9@".getBytes());
+ GenericRecord r = serializeAndDeserialize(field, "fixed1", fixed);
+
+ assertArrayEquals(fixed.bytes(), ((GenericData.Fixed) r.get("fixed1")).bytes());
+ }
+
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestGenericAvroRecordWritable {
+ private static final String schemaJSON = "{\n" +
+ " \"namespace\": \"gallifrey\",\n" +
+ " \"name\": \"TestPerson\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"first\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"last\",\n" +
+ " \"type\":\"string\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ @Test
+ public void writableContractIsImplementedCorrectly() throws IOException {
+ Schema schema = Schema.parse(schemaJSON);
+
+ GenericRecord gr = new GenericData.Record(schema);
+ gr.put("first", "The");
+ gr.put("last", "Doctor");
+
+ assertEquals("The", gr.get("first"));
+ assertEquals("Doctor", gr.get("last"));
+
+ AvroGenericRecordWritable garw = new AvroGenericRecordWritable(gr);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream daos = new DataOutputStream(baos);
+ garw.write(daos);
+
+ AvroGenericRecordWritable garw2 = new AvroGenericRecordWritable(gr);
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dais = new DataInputStream(bais);
+
+ garw2.readFields(dais);
+
+ GenericRecord gr2 = garw2.getRecord();
+
+ assertEquals("The", gr2.get("first").toString());
+ assertEquals("Doctor", gr2.get("last").toString());
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertSame;
+
+public class TestInstanceCache {
+ private static class Foo {
+ @Override
+ public int hashCode() {
+ return 42;
+ }
+ }
+
+ private static class Wrapper<T> {
+ public final T wrapped;
+
+ private Wrapper(T wrapped) {
+ this.wrapped = wrapped;
+ }
+ }
+
+ @Test
+ public void instanceCachesOnlyCreateOneInstance() throws AvroSerdeException {
+ InstanceCache<Foo, Wrapper<Foo>> ic = new InstanceCache<Foo, Wrapper<Foo>>() {
+ @Override
+ protected Wrapper makeInstance(Foo hv) {
+ return new Wrapper(hv);
+ }
+ };
+ Foo f1 = new Foo();
+
+ Wrapper fc = ic.retrieve(f1);
+ assertSame(f1, fc.wrapped); // Our original foo should be in the wrapper
+
+ Foo f2 = new Foo(); // Different instance, same value
+
+ Wrapper fc2 = ic.retrieve(f2);
+ assertSame(fc2,fc); // Since equiv f, should get back first container
+ assertSame(fc2.wrapped, f1);
+ }
+
+ @Test
+ public void instanceCacheReturnsCorrectInstances() throws AvroSerdeException {
+ InstanceCache<String, Wrapper<String>> ic = new InstanceCache<String, Wrapper<String>>() {
+ @Override
+ protected Wrapper<String> makeInstance(String hv) {
+ return new Wrapper<String>(hv);
+ }
+ };
+
+ Wrapper<String> one = ic.retrieve("one");
+ Wrapper<String> two = ic.retrieve("two");
+
+ Wrapper<String> anotherOne = ic.retrieve("one");
+ assertSame(one, anotherOne);
+
+ Wrapper<String> anotherTwo = ic.retrieve("two");
+ assertSame(two, anotherTwo);
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestSchemaReEncoder {
+ @Test
+ public void schemasCanAddFields() throws SerDeException {
+ String original = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"Line\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"text\",\n" +
+ " \"type\":\"string\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ String evolved = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"Line\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"text\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"new_kid\",\n" +
+ " \"type\":\"string\",\n" +
+ " \"default\":\"Hi!\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ Schema originalSchema = Schema.parse(original);
+ Schema evolvedSchema = Schema.parse(evolved);
+
+ GenericRecord record = new GenericData.Record(originalSchema);
+ record.put("text", "it is a far better thing I do, yadda, yadda");
+ assertTrue(GenericData.get().validate(originalSchema, record));
+ AvroDeserializer.SchemaReEncoder schemaReEncoder = new AvroDeserializer.SchemaReEncoder();
+ GenericRecord r2 = schemaReEncoder.reencode(record, evolvedSchema);
+
+ assertTrue(GenericData.get().validate(evolvedSchema, r2));
+ assertEquals("Hi!", r2.get("new_kid").toString());
+
+ // Now make sure that we can re-use the re-encoder against a completely
+ // different record to save resources
+ String original2 = "{\n" +
+ " \"namespace\": \"somebody.else\",\n" +
+ " \"name\": \"something_else\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"a\",\n" +
+ " \"type\":\"int\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ String evolved2 = "{\n" +
+ " \"namespace\": \"somebody.else\",\n" +
+ " \"name\": \"something_else\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"a\",\n" +
+ " \"type\":\"int\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"b\",\n" +
+ " \"type\":\"long\",\n" +
+ " \"default\":42\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ Schema originalSchema2 = Schema.parse(original2);
+ Schema evolvedSchema2 = Schema.parse(evolved2);
+
+ record = new GenericData.Record(originalSchema2);
+ record.put("a", 19);
+ assertTrue(GenericData.get().validate(originalSchema2, record));
+
+ r2 = schemaReEncoder.reencode(record, evolvedSchema2);
+ assertTrue(GenericData.get().validate(evolvedSchema2, r2));
+ assertEquals(42l, r2.get("b"));
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestThatEvolvedSchemasActAsWeWant {
+ @Test
+ public void resolvedSchemasShouldReturnReaderSchema() throws IOException {
+ // Need to verify that when reading a datum with an updated reader schema
+ // that the datum then returns the reader schema as its own, since we
+ // depend on this behavior in order to avoid re-encoding the datum
+ // in the serde.
+ String v0 = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"SomeStuff\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"v0\",\n" +
+ " \"type\":\"string\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ String v1 = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"SomeStuff\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"v0\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"v1\",\n" +
+ " \"type\":\"string\",\n" +
+ " \"default\":\"v1_default\"" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ Schema[] schemas = {Schema.parse(v0), Schema.parse(v1)};
+
+ // Encode a schema with v0, write out.
+ GenericRecord record = new GenericData.Record(schemas[0]);
+ record.put("v0", "v0 value");
+ assertTrue(GenericData.get().validate(schemas[0], record));
+
+ // Write datum out to a stream
+ GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schemas[0]);
+ DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ dfw.create(schemas[0], baos);
+ dfw.append(record);
+ dfw.close();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();
+ gdr.setExpected(schemas[1]);
+ DataFileStream<GenericRecord> dfs = new DataFileStream<GenericRecord>(bais, gdr);
+ assertTrue(dfs.hasNext());
+ GenericRecord next = dfs.next();
+ assertEquals("v0 value", next.get("v0").toString());
+ assertEquals("v1_default", next.get("v1").toString());
+
+ // Now the most important check - when we query this record for its schema,
+ // we should get back the latest, reader schema:
+ assertEquals(schemas[1], next.getSchema());
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java?rev=1345420&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java Sat Jun 2 02:14:04 2012
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.generic.GenericData;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+class Utils {
+ // Force Avro to serialize and de-serialize the record to make sure it has a
+ // chance to muck with the bytes and we're working against real Avro data.
+ public static AvroGenericRecordWritable
+ serializeAndDeserializeRecord(GenericData.Record record) throws IOException {
+ AvroGenericRecordWritable garw = new AvroGenericRecordWritable(record);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream daos = new DataOutputStream(baos);
+ garw.write(daos);
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dais = new DataInputStream(bais);
+
+ AvroGenericRecordWritable garw2 = new AvroGenericRecordWritable();
+ garw2.readFields(dais);
+ return garw2;
+ }
+}