You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by zi...@apache.org on 2019/05/30 12:12:29 UTC
[parquet-mr] branch master updated: PARQUET-1576 Bump Apache Avro
to 1.9.0 (#638)
This is an automated email from the ASF dual-hosted git repository.
zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 9d6fb45 PARQUET-1576 Bump Apache Avro to 1.9.0 (#638)
9d6fb45 is described below
commit 9d6fb45e54da65cbd407bb3e7bff0981aa9f8f9f
Author: Fokko Driesprong <fo...@driesprong.frl>
AuthorDate: Thu May 30 14:12:24 2019 +0200
PARQUET-1576 Bump Apache Avro to 1.9.0 (#638)
---
.../java/org/apache/parquet/avro/AvroTestUtil.java | 5 ++--
.../parquet/avro/TestAvroSchemaConverter.java | 22 +++++++--------
.../parquet/avro/TestReflectLogicalTypes.java | 7 ++---
.../apache/parquet/avro/TestSpecificReadWrite.java | 32 ++++++++++++----------
pom.xml | 2 +-
5 files changed, 35 insertions(+), 33 deletions(-)
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
index 801f2bd..717e36e 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+
+import org.apache.avro.JsonProperties;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
@@ -31,7 +33,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.ParquetWriter;
-import org.codehaus.jackson.node.NullNode;
import org.junit.Assert;
import org.junit.rules.TemporaryFolder;
@@ -52,7 +53,7 @@ public class AvroTestUtil {
}
public static Schema.Field optionalField(String name, Schema schema) {
- return new Schema.Field(name, optional(schema), null, NullNode.getInstance());
+ return new Schema.Field(name, optional(schema), null, JsonProperties.NULL_VALUE);
}
public static Schema array(Schema element) {
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
index 2548408..36bd7ba 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
@@ -20,6 +20,7 @@ package org.apache.parquet.avro;
import com.google.common.collect.Lists;
import com.google.common.io.Resources;
+import org.apache.avro.JsonProperties;
import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
@@ -28,7 +29,6 @@ import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Types;
-import org.codehaus.jackson.node.NullNode;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -296,8 +296,8 @@ public class TestAvroSchemaConverter {
public void testOptionalFields() throws Exception {
Schema schema = Schema.createRecord("record1", null, null, false);
Schema optionalInt = optional(Schema.create(INT));
- schema.setFields(Arrays.asList(
- new Schema.Field("myint", optionalInt, null, NullNode.getInstance())
+ schema.setFields(Collections.singletonList(
+ new Schema.Field("myint", optionalInt, null, JsonProperties.NULL_VALUE)
));
testRoundTripConversion(
schema,
@@ -351,7 +351,7 @@ public class TestAvroSchemaConverter {
Schema.create(INT),
Schema.create(Schema.Type.FLOAT)));
schema.setFields(Arrays.asList(
- new Schema.Field("myunion", multipleTypes, null, NullNode.getInstance())));
+ new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));
// Avro union is modelled using optional data members of the different
// types. This does not translate back into an Avro union
@@ -370,8 +370,8 @@ public class TestAvroSchemaConverter {
Schema innerRecord = Schema.createRecord("element", null, null, false);
Schema optionalString = optional(Schema.create(Schema.Type.STRING));
innerRecord.setFields(Lists.newArrayList(
- new Schema.Field("s1", optionalString, null, NullNode.getInstance()),
- new Schema.Field("s2", optionalString, null, NullNode.getInstance())
+ new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
+ new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
));
Schema schema = Schema.createRecord("HasArray", null, null, false);
schema.setFields(Lists.newArrayList(
@@ -397,8 +397,8 @@ public class TestAvroSchemaConverter {
Schema innerRecord = Schema.createRecord("InnerRecord", null, null, false);
Schema optionalString = optional(Schema.create(Schema.Type.STRING));
innerRecord.setFields(Lists.newArrayList(
- new Schema.Field("s1", optionalString, null, NullNode.getInstance()),
- new Schema.Field("s2", optionalString, null, NullNode.getInstance())
+ new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
+ new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
));
Schema schema = Schema.createRecord("HasArray", null, null, false);
schema.setFields(Lists.newArrayList(
@@ -424,7 +424,7 @@ public class TestAvroSchemaConverter {
Schema.create(INT))));
Schema schema = Schema.createRecord("AvroCompatListInList", null, null, false);
schema.setFields(Lists.newArrayList(
- new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+ new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
));
System.err.println("Avro schema: " + schema.toString(true));
@@ -453,7 +453,7 @@ public class TestAvroSchemaConverter {
Schema.create(INT))));
Schema schema = Schema.createRecord("ThriftCompatListInList", null, null, false);
schema.setFields(Lists.newArrayList(
- new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+ new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
));
System.err.println("Avro schema: " + schema.toString(true));
@@ -486,7 +486,7 @@ public class TestAvroSchemaConverter {
Schema.create(INT))));
Schema schema = Schema.createRecord("UnknownTwoLevelListInList", null, null, false);
schema.setFields(Lists.newArrayList(
- new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+ new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
));
System.err.println("Avro schema: " + schema.toString(true));
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
index 401e698..ede5209 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
@@ -120,7 +120,7 @@ public class TestReflectLogicalTypes {
public void testDecimalBytes() throws IOException {
Schema schema = REFLECT.getSchema(DecimalRecordBytes.class);
Assert.assertEquals("Should have the correct record name",
- "org.apache.parquet.avro.TestReflectLogicalTypes$",
+ "org.apache.parquet.avro.TestReflectLogicalTypes",
schema.getNamespace());
Assert.assertEquals("Should have the correct record name",
"DecimalRecordBytes",
@@ -179,7 +179,7 @@ public class TestReflectLogicalTypes {
public void testDecimalFixed() throws IOException {
Schema schema = REFLECT.getSchema(DecimalRecordFixed.class);
Assert.assertEquals("Should have the correct record name",
- "org.apache.parquet.avro.TestReflectLogicalTypes$",
+ "org.apache.parquet.avro.TestReflectLogicalTypes",
schema.getNamespace());
Assert.assertEquals("Should have the correct record name",
"DecimalRecordFixed",
@@ -260,7 +260,6 @@ public class TestReflectLogicalTypes {
}
@Test
- @SuppressWarnings("unchecked")
public void testPairRecord() throws IOException {
ReflectData model = new ReflectData();
model.addLogicalTypeConversion(new Conversion<Pair>() {
@@ -298,7 +297,7 @@ public class TestReflectLogicalTypes {
Schema schema = model.getSchema(PairRecord.class);
Assert.assertEquals("Should have the correct record name",
- "org.apache.parquet.avro.TestReflectLogicalTypes$",
+ "org.apache.parquet.avro.TestReflectLogicalTypes",
schema.getNamespace());
Assert.assertEquals("Should have the correct record name",
"PairRecord",
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
index fa37d54..46e2f2c 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,8 @@
package org.apache.parquet.avro;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import static org.apache.parquet.filter.ColumnPredicates.equalTo;
import static org.apache.parquet.filter.ColumnRecordFilter.column;
@@ -205,12 +207,12 @@ public class TestSpecificReadWrite {
try(ParquetReader<Car> reader = new AvroParquetReader<Car>(conf, path)) {
for (Car car = reader.read(); car != null; car = reader.read()) {
- assertEquals(car.getDoors() != null, true);
- assertEquals(car.getEngine() != null, true);
- assertEquals(car.getMake() != null, true);
- assertEquals(car.getModel() != null, true);
- assertEquals(car.getYear() != null, true);
- assertEquals(car.getVin() != null, true);
+ assertTrue(car.getDoors() == 4 || car.getDoors() == 5);
+ assertNotNull(car.getEngine());
+ assertNotNull(car.getMake());
+ assertNotNull(car.getModel());
+ assertEquals(2010, car.getYear());
+ assertNotNull(car.getVin());
assertNull(car.getOptionalExtra());
assertNull(car.getServiceHistory());
}
@@ -225,12 +227,12 @@ public class TestSpecificReadWrite {
try(ParquetReader<NewCar> reader = new AvroParquetReader<>(conf, path)) {
for (NewCar car = reader.read(); car != null; car = reader.read()) {
- assertEquals(car.getEngine() != null, true);
- assertEquals(car.getBrand() != null, true);
- assertEquals(car.getYear() != null, true);
- assertEquals(car.getVin() != null, true);
- assertEquals(car.getDescription() == null, true);
- assertEquals(car.getOpt() == 5, true);
+ assertNotNull(car.getEngine());
+ assertNotNull(car.getBrand());
+ assertEquals(2010, car.getYear());
+ assertNotNull(car.getVin());
+ assertNull(car.getDescription());
+ assertEquals(5, car.getOpt());
}
}
}
diff --git a/pom.xml b/pom.xml
index 0d52add..f8be2ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,7 +97,7 @@
<fastutil.version>7.0.13</fastutil.version>
<semver.api.version>0.9.33</semver.api.version>
<slf4j.version>1.7.22</slf4j.version>
- <avro.version>1.8.2</avro.version>
+ <avro.version>1.9.0</avro.version>
<guava.version>27.0.1-jre</guava.version>
<brotli-codec.version>0.1.1</brotli-codec.version>
<mockito.version>1.10.19</mockito.version>