You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by nk...@apache.org on 2019/07/25 13:15:41 UTC
[parquet-mr] branch master updated: PARQUET-1303 correct
ClassCastException for Avro @Stringable fields (#482)
This is an automated email from the ASF dual-hosted git repository.
nkollar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 93af6b4 PARQUET-1303 correct ClassCastException for Avro @Stringable fields (#482)
93af6b4 is described below
commit 93af6b4e8db84c813530cda763f378858e8c7700
Author: Zack Behringer <ze...@gmail.com>
AuthorDate: Thu Jul 25 09:15:35 2019 -0400
PARQUET-1303 correct ClassCastException for Avro @Stringable fields (#482)
---
.../org/apache/parquet/avro/AvroWriteSupport.java | 4 +++-
.../parquet/avro/TestReflectLogicalTypes.java | 18 ++++++++++++----
.../apache/parquet/avro/TestReflectReadWrite.java | 24 +++++++++++++++++++++-
3 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
index 2325dc5..859a133 100644
--- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
+++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
@@ -367,8 +367,10 @@ public class AvroWriteSupport<T> extends WriteSupport<T> {
if (value instanceof Utf8) {
Utf8 utf8 = (Utf8) value;
return Binary.fromReusedByteArray(utf8.getBytes(), 0, utf8.getByteLength());
+ } else if (value instanceof CharSequence) {
+ return Binary.fromCharSequence((CharSequence) value);
}
- return Binary.fromCharSequence((CharSequence) value);
+ return Binary.fromCharSequence(value.toString());
}
private static GenericData getDataModel(Configuration conf) {
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
index ede5209..142191f 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
@@ -425,7 +425,7 @@ public class TestReflectLogicalTypes {
read(REFLECT, nullableUuidStringSchema, test));
}
- @Test(expected = ClassCastException.class)
+ @Test
public void testWriteUUIDMissingLogicalType() throws IOException {
Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName())
.fields().requiredString("uuid").endRecord();
@@ -439,6 +439,11 @@ public class TestReflectLogicalTypes {
RecordWithUUID r2 = new RecordWithUUID();
r2.uuid = u2;
+ List<RecordWithStringUUID> expected = Arrays.asList(
+ new RecordWithStringUUID(), new RecordWithStringUUID());
+ expected.get(0).uuid = u1.toString();
+ expected.get(1).uuid = u2.toString();
+
// write without using REFLECT, which has the logical type
File test = write(uuidSchema, r1, r2);
@@ -447,9 +452,14 @@ public class TestReflectLogicalTypes {
.record(RecordWithStringUUID.class.getName())
.fields().requiredString("uuid").endRecord();
- // this fails with an AppendWriteException wrapping ClassCastException
- // because the UUID isn't converted to a CharSequence expected internally
- read(ReflectData.get(), uuidStringSchema, test);
+ Assert.assertEquals("Should read uuid as String without UUID conversion",
+ expected,
+ read(REFLECT, uuidStringSchema, test));
+
+ Assert.assertEquals("Should read uuid as String without UUID logical type",
+ expected,
+ read(ReflectData.get(), uuidStringSchema, test)
+ );
}
@Test
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
index 2c88d05..539427b 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
@@ -30,6 +30,7 @@ import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.Stringable;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -101,6 +102,7 @@ public class TestReflectReadWrite {
schema.getField("mystringarray").schema(), Lists.newArrayList(new Utf8("a"), new Utf8("b"))));
record.put("mylist", new GenericData.Array<Utf8>(
schema.getField("mylist").schema(), Lists.newArrayList(new Utf8("a"), new Utf8("b"), new Utf8("c"))));
+ record.put("mystringable", new StringableObj("blah blah"));
return record;
}
@@ -124,6 +126,7 @@ public class TestReflectReadWrite {
object.myintarray = new int[] { 1, 2 };
object.mystringarray = new String[] { "a", "b" };
object.mylist = Lists.newArrayList("a", "b", "c");
+ object.mystringable = new StringableObj("blah blah");
return object;
}
@@ -154,6 +157,21 @@ public class TestReflectReadWrite {
A, B
}
+ public static class StringableObj {
+ private String value;
+ public StringableObj(String value) {
+ this.value = value;
+ }
+ @Override
+ public String toString() {
+ return this.value;
+ }
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof StringableObj && this.value.equals(((StringableObj)other).value);
+ }
+ }
+
public static class Pojo {
public boolean myboolean;
public byte mybyte;
@@ -171,6 +189,8 @@ public class TestReflectReadWrite {
private int[] myintarray;
private String[] mystringarray;
private List<String> mylist;
+ @Stringable
+ private StringableObj mystringable;
@Override
public boolean equals(Object o) {
@@ -190,7 +210,8 @@ public class TestReflectReadWrite {
&& Arrays.equals(myshortarray, that.myshortarray)
&& Arrays.equals(myintarray, that.myintarray)
&& Arrays.equals(mystringarray, that.mystringarray)
- && mylist.equals(that.mylist);
+ && mylist.equals(that.mylist)
+ && mystringable.equals(that.mystringable);
}
@Override
@@ -211,6 +232,7 @@ public class TestReflectReadWrite {
", myintarray=" + Arrays.toString(myintarray) +
", mystringarray=" + Arrays.toString(mystringarray) +
", mylist=" + mylist +
+ ", mystringable=" + mystringable.toString() +
'}';
}
}