You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by zi...@apache.org on 2019/05/30 12:12:04 UTC
[parquet-mr] branch master updated: PARQUET-1556 Use
try-with-resource in Apache Avro tests (#639)
This is an automated email from the ASF dual-hosted git repository.
zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 54777b4 PARQUET-1556 Use try-with-resource in Apache Avro tests (#639)
54777b4 is described below
commit 54777b46da768df3ff88548c6b6b70bd52fba0dd
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Thu May 30 14:11:58 2019 +0200
PARQUET-1556 Use try-with-resource in Apache Avro tests (#639)
---
.../java/org/apache/parquet/avro/AvroTestUtil.java | 26 ++---
.../parquet/avro/TestArrayCompatibility.java | 9 +-
.../org/apache/parquet/avro/TestReadWrite.java | 99 ++++++++--------
.../parquet/avro/TestReadWriteOldListBehavior.java | 128 ++++++++++-----------
4 files changed, 131 insertions(+), 131 deletions(-)
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
index 39c6d2a..801f2bd 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
@@ -85,19 +85,16 @@ public class AvroTestUtil {
Configuration conf = new Configuration(false);
AvroReadSupport.setRequestedProjection(conf, schema);
AvroReadSupport.setAvroReadSchema(conf, schema);
- ParquetReader<D> fileReader = AvroParquetReader
- .<D>builder(new Path(file.toString()))
- .withDataModel(model) // reflect disables compatibility
- .withConf(conf)
- .build();
- try {
+ try (ParquetReader<D> fileReader = AvroParquetReader
+ .<D>builder(new Path(file.toString()))
+ .withDataModel(model) // reflect disables compatibility
+ .withConf(conf)
+ .build()) {
D datum;
while ((datum = fileReader.read()) != null) {
data.add(datum);
}
- } finally {
- fileReader.close();
}
return data;
@@ -107,18 +104,15 @@ public class AvroTestUtil {
public static <D> File write(TemporaryFolder temp, GenericData model, Schema schema, D... data) throws IOException {
File file = temp.newFile();
Assert.assertTrue(file.delete());
- ParquetWriter<D> writer = AvroParquetWriter
- .<D>builder(new Path(file.toString()))
- .withDataModel(model)
- .withSchema(schema)
- .build();
- try {
+ try (ParquetWriter<D> writer = AvroParquetWriter
+ .<D>builder(new Path(file.toString()))
+ .withDataModel(model)
+ .withSchema(schema)
+ .build()) {
for (D datum : data) {
writer.write(datum);
}
- } finally {
- writer.close();
}
return file;
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
index aa577ab..1f47ba7 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
@@ -1114,12 +1114,13 @@ public class TestArrayCompatibility extends DirectWriterTest {
instance(structWithElementField, "element", 34.0F))));
// check the schema
- ParquetFileReader reader = ParquetFileReader
- .open(new Configuration(), test);
- MessageType fileSchema = reader.getFileMetaData().getSchema();
- Assert.assertEquals("Converted schema should assume 2-layer structure",
+ final MessageType fileSchema;
+ try (ParquetFileReader reader = ParquetFileReader.open(new Configuration(), test)) {
+ fileSchema = reader.getFileMetaData().getSchema();
+ Assert.assertEquals("Converted schema should assume 2-layer structure",
oldSchema,
new AvroSchemaConverter(OLD_BEHAVIOR_CONF).convert(fileSchema));
+ }
// both should default to the 2-layer structure
assertReaderContains(oldBehaviorReader(test), oldSchema, oldRecord);
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
index 60ff269..1e726c3 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
@@ -89,26 +89,27 @@ public class TestReadWrite {
Schema schema = new Schema.Parser().parse(
Resources.getResource("array.avsc").openStream());
+ // Write a record with an empty array.
+ List<Integer> emptyArray = new ArrayList<>();
+
Path file = new Path(createTempFile().getPath());
- ParquetWriter<GenericRecord> writer = AvroParquetWriter
+ try(ParquetWriter<GenericRecord> writer = AvroParquetWriter
.<GenericRecord>builder(file)
.withSchema(schema)
.withConf(testConf)
- .build();
-
- // Write a record with an empty array.
- List<Integer> emptyArray = new ArrayList<Integer>();
- GenericData.Record record = new GenericRecordBuilder(schema)
+ .build()) {
+ GenericData.Record record = new GenericRecordBuilder(schema)
.set("myarray", emptyArray).build();
- writer.write(record);
- writer.close();
+ writer.write(record);
+ }
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try (AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(emptyArray, nextRecord.get("myarray"));
+ assertNotNull(nextRecord);
+ assertEquals(emptyArray, nextRecord.get("myarray"));
+ }
}
@Test
@@ -131,11 +132,12 @@ public class TestReadWrite {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(emptyMap, nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(emptyMap, nextRecord.get("mymap"));
+ }
}
@Test
@@ -162,11 +164,12 @@ public class TestReadWrite {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(map, nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(map, nextRecord.get("mymap"));
+ }
}
@Test(expected=RuntimeException.class)
@@ -215,11 +218,12 @@ public class TestReadWrite {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ }
}
@Rule
@@ -606,25 +610,25 @@ public class TestReadWrite {
GenericFixed genericFixed = new GenericData.Fixed(
Schema.createFixed("fixed", null, null, 1), new byte[] { (byte) 65 });
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(true, nextRecord.get("myboolean"));
- assertEquals(1, nextRecord.get("myint"));
- assertEquals(2L, nextRecord.get("mylong"));
- assertEquals(3.1f, nextRecord.get("myfloat"));
- assertEquals(4.1, nextRecord.get("mydouble"));
- assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
- assertEquals(str("hello"), nextRecord.get("mystring"));
- assertEquals(str("a"), nextRecord.get("myenum")); // enum symbols are unknown
- assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
- assertEquals(integerArray, nextRecord.get("myarray"));
- assertEquals(integerArray, nextRecord.get("myoptionalarray"));
- assertEquals(ingeterArrayWithNulls, nextRecord.get("myarrayofoptional"));
- assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
- assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
- assertEquals(genericFixed, nextRecord.get("myfixed"));
-
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
+ assertNotNull(nextRecord);
+ assertEquals(true, nextRecord.get("myboolean"));
+ assertEquals(1, nextRecord.get("myint"));
+ assertEquals(2L, nextRecord.get("mylong"));
+ assertEquals(3.1f, nextRecord.get("myfloat"));
+ assertEquals(4.1, nextRecord.get("mydouble"));
+ assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+ assertEquals(str("hello"), nextRecord.get("mystring"));
+ assertEquals(str("a"), nextRecord.get("myenum")); // enum symbols are unknown
+ assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+ assertEquals(integerArray, nextRecord.get("myarray"));
+ assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+ assertEquals(ingeterArrayWithNulls, nextRecord.get("myarrayofoptional"));
+ assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
+ assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ assertEquals(genericFixed, nextRecord.get("myfixed"));
+ }
}
@Test
@@ -648,11 +652,12 @@ public class TestReadWrite {
parquetWriter.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(str("theValue"), nextRecord.get("value"));
+ assertNotNull(nextRecord);
+ assertEquals(str("theValue"), nextRecord.get("value"));
+ }
}
@Test
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
index 4542ef2..f12417c 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
@@ -94,11 +94,12 @@ public class TestReadWriteOldListBehavior {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(emptyArray, nextRecord.get("myarray"));
+ assertNotNull(nextRecord);
+ assertEquals(emptyArray, nextRecord.get("myarray"));
+ }
}
@Test
@@ -119,11 +120,12 @@ public class TestReadWriteOldListBehavior {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(emptyMap, nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(emptyMap, nextRecord.get("mymap"));
+ }
}
@Test
@@ -148,11 +150,12 @@ public class TestReadWriteOldListBehavior {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(map, nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(map, nextRecord.get("mymap"));
+ }
}
@Test(expected=RuntimeException.class)
@@ -197,11 +200,12 @@ public class TestReadWriteOldListBehavior {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ assertNotNull(nextRecord);
+ assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ }
}
@Test
@@ -252,30 +256,31 @@ public class TestReadWriteOldListBehavior {
writer.write(record);
}
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
- Object expectedEnumSymbol = compat ? "a" :
+ Object expectedEnumSymbol = compat ? "a" :
new GenericData.EnumSymbol(schema.getField("myenum").schema(), "a");
- assertNotNull(nextRecord);
- assertEquals(null, nextRecord.get("mynull"));
- assertEquals(true, nextRecord.get("myboolean"));
- assertEquals(1, nextRecord.get("myint"));
- assertEquals(2L, nextRecord.get("mylong"));
- assertEquals(3.1f, nextRecord.get("myfloat"));
- assertEquals(4.1, nextRecord.get("mydouble"));
- assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
- assertEquals(str("hello"), nextRecord.get("mystring"));
- assertEquals(expectedEnumSymbol, nextRecord.get("myenum"));
- assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
- assertEquals(integerArray, nextRecord.get("myarray"));
- assertEquals(emptyArray, nextRecord.get("myemptyarray"));
- assertEquals(integerArray, nextRecord.get("myoptionalarray"));
- assertEquals(integerArray, nextRecord.get("myarrayofoptional"));
- assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
- assertEquals(emptyMap, nextRecord.get("myemptymap"));
- assertEquals(genericFixed, nextRecord.get("myfixed"));
+ assertNotNull(nextRecord);
+ assertEquals(null, nextRecord.get("mynull"));
+ assertEquals(true, nextRecord.get("myboolean"));
+ assertEquals(1, nextRecord.get("myint"));
+ assertEquals(2L, nextRecord.get("mylong"));
+ assertEquals(3.1f, nextRecord.get("myfloat"));
+ assertEquals(4.1, nextRecord.get("mydouble"));
+ assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+ assertEquals(str("hello"), nextRecord.get("mystring"));
+ assertEquals(expectedEnumSymbol, nextRecord.get("myenum"));
+ assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+ assertEquals(integerArray, nextRecord.get("myarray"));
+ assertEquals(emptyArray, nextRecord.get("myemptyarray"));
+ assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+ assertEquals(integerArray, nextRecord.get("myarrayofoptional"));
+ assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ assertEquals(emptyMap, nextRecord.get("myemptymap"));
+ assertEquals(genericFixed, nextRecord.get("myfixed"));
+ }
}
@Test
@@ -305,7 +310,7 @@ public class TestReadWriteOldListBehavior {
Schema arrayOfOptionalIntegers = Schema.createArray(
optional(Schema.create(Schema.Type.INT)));
GenericData.Array<Integer> genericIntegerArrayWithNulls =
- new GenericData.Array<Integer>(
+ new GenericData.Array<>(
arrayOfOptionalIntegers,
Arrays.asList(1, null, 2, null, 3));
@@ -329,17 +334,12 @@ public class TestReadWriteOldListBehavior {
.set("myfixed", genericFixed)
.build();
- final AvroParquetWriter<GenericRecord> writer =
- new AvroParquetWriter<GenericRecord>(file, schema);
-
- try {
+ try (AvroParquetWriter<GenericRecord> writer = new AvroParquetWriter<>(file, schema)) {
writer.write(record);
fail("Should not succeed writing an array with null values");
} catch (Exception e) {
Assert.assertTrue("Error message should provide context and help",
- e.getMessage().contains("parquet.avro.write-old-list-structure"));
- } finally {
- writer.close();
+ e.getMessage().contains("parquet.avro.write-old-list-structure"));
}
}
@@ -554,25 +554,25 @@ public class TestReadWriteOldListBehavior {
elementBuilder.set("element", 3).build()
));
- AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
- GenericRecord nextRecord = reader.read();
- assertNotNull(nextRecord);
- assertEquals(true, nextRecord.get("myboolean"));
- assertEquals(1, nextRecord.get("myint"));
- assertEquals(2L, nextRecord.get("mylong"));
- assertEquals(3.1f, nextRecord.get("myfloat"));
- assertEquals(4.1, nextRecord.get("mydouble"));
- assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
- assertEquals(str("hello"), nextRecord.get("mystring"));
- assertEquals(str("a"), nextRecord.get("myenum"));
- assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
- assertEquals(integerArray, nextRecord.get("myarray"));
- assertEquals(integerArray, nextRecord.get("myoptionalarray"));
- assertEquals(genericRecordArrayWithNullIntegers, nextRecord.get("myarrayofoptional"));
- assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
- assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
- assertEquals(genericFixed, nextRecord.get("myfixed"));
-
+ try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf, file)) {
+ GenericRecord nextRecord = reader.read();
+ assertNotNull(nextRecord);
+ assertEquals(true, nextRecord.get("myboolean"));
+ assertEquals(1, nextRecord.get("myint"));
+ assertEquals(2L, nextRecord.get("mylong"));
+ assertEquals(3.1f, nextRecord.get("myfloat"));
+ assertEquals(4.1, nextRecord.get("mydouble"));
+ assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+ assertEquals(str("hello"), nextRecord.get("mystring"));
+ assertEquals(str("a"), nextRecord.get("myenum"));
+ assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+ assertEquals(integerArray, nextRecord.get("myarray"));
+ assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+ assertEquals(genericRecordArrayWithNullIntegers, nextRecord.get("myarrayofoptional"));
+ assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
+ assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+ assertEquals(genericFixed, nextRecord.get("myfixed"));
+ }
}
/**