You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/07/30 14:30:55 UTC
spark git commit: [SPARK-24771][BUILD] Upgrade Apache AVRO to 1.8.2
Repository: spark
Updated Branches:
refs/heads/master fca0b8528 -> b90bfe3c4
[SPARK-24771][BUILD] Upgrade Apache AVRO to 1.8.2
## What changes were proposed in this pull request?
Upgrade Apache Avro from 1.7.7 to 1.8.2. The major new features:
1. More logical types. From the spec of 1.8.2 https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types we can see comparing to [1.7.7](https://avro.apache.org/docs/1.7.7/spec.html#Logical+Types), the new version support:
- Date
- Time (millisecond precision)
- Time (microsecond precision)
- Timestamp (millisecond precision)
- Timestamp (microsecond precision)
- Duration
2. Single-object encoding: https://avro.apache.org/docs/1.8.2/spec.html#single_object_encoding
This PR aims to update Apache Spark to support these new features.
## How was this patch tested?
Unit test
Author: Gengliang Wang <ge...@databricks.com>
Closes #21761 from gengliangwang/upgrade_avro_1.8.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b90bfe3c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b90bfe3c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b90bfe3c
Branch: refs/heads/master
Commit: b90bfe3c42eb9b51e6131a8f8923bcddfccd75bb
Parents: fca0b85
Author: Gengliang Wang <ge...@databricks.com>
Authored: Mon Jul 30 07:30:47 2018 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Mon Jul 30 07:30:47 2018 -0700
----------------------------------------------------------------------
dev/deps/spark-deps-hadoop-2.6 | 10 +++++-----
dev/deps/spark-deps-hadoop-2.7 | 10 +++++-----
dev/deps/spark-deps-hadoop-3.1 | 10 +++++-----
pom.xml | 2 +-
sql/core/pom.xml | 13 -------------
5 files changed, 16 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-2.6
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index ff6d5c3..4ef61b2 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -18,9 +18,9 @@ arrow-format-0.8.0.jar
arrow-memory-0.8.0.jar
arrow-vector-0.8.0.jar
automaton-1.11-8.jar
-avro-1.7.7.jar
-avro-ipc-1.7.7.jar
-avro-mapred-1.7.7-hadoop2.jar
+avro-1.8.2.jar
+avro-ipc-1.8.2.jar
+avro-mapred-1.8.2-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.58.jar
bonecp-0.8.0.RELEASE.jar
@@ -37,7 +37,7 @@ commons-cli-1.2.jar
commons-codec-1.10.jar
commons-collections-3.2.2.jar
commons-compiler-3.0.8.jar
-commons-compress-1.4.1.jar
+commons-compress-1.8.1.jar
commons-configuration-1.6.jar
commons-crypto-1.0.0.jar
commons-dbcp-1.4.jar
@@ -196,7 +196,7 @@ validation-api-1.1.0.Final.jar
xbean-asm6-shaded-4.8.jar
xercesImpl-2.9.1.jar
xmlenc-0.52.jar
-xz-1.0.jar
+xz-1.5.jar
zjsonpatch-0.3.0.jar
zookeeper-3.4.6.jar
zstd-jni-1.3.2-2.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-2.7
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 72a94f8..a74ce1f 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -18,9 +18,9 @@ arrow-format-0.8.0.jar
arrow-memory-0.8.0.jar
arrow-vector-0.8.0.jar
automaton-1.11-8.jar
-avro-1.7.7.jar
-avro-ipc-1.7.7.jar
-avro-mapred-1.7.7-hadoop2.jar
+avro-1.8.2.jar
+avro-ipc-1.8.2.jar
+avro-mapred-1.8.2-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.58.jar
bonecp-0.8.0.RELEASE.jar
@@ -37,7 +37,7 @@ commons-cli-1.2.jar
commons-codec-1.10.jar
commons-collections-3.2.2.jar
commons-compiler-3.0.8.jar
-commons-compress-1.4.1.jar
+commons-compress-1.8.1.jar
commons-configuration-1.6.jar
commons-crypto-1.0.0.jar
commons-dbcp-1.4.jar
@@ -197,7 +197,7 @@ validation-api-1.1.0.Final.jar
xbean-asm6-shaded-4.8.jar
xercesImpl-2.9.1.jar
xmlenc-0.52.jar
-xz-1.0.jar
+xz-1.5.jar
zjsonpatch-0.3.0.jar
zookeeper-3.4.6.jar
zstd-jni-1.3.2-2.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-3.1
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1
index 3409dc4..e0fcca0 100644
--- a/dev/deps/spark-deps-hadoop-3.1
+++ b/dev/deps/spark-deps-hadoop-3.1
@@ -16,9 +16,9 @@ arrow-format-0.8.0.jar
arrow-memory-0.8.0.jar
arrow-vector-0.8.0.jar
automaton-1.11-8.jar
-avro-1.7.7.jar
-avro-ipc-1.7.7.jar
-avro-mapred-1.7.7-hadoop2.jar
+avro-1.8.2.jar
+avro-ipc-1.8.2.jar
+avro-mapred-1.8.2-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.58.jar
bonecp-0.8.0.RELEASE.jar
@@ -34,7 +34,7 @@ commons-cli-1.2.jar
commons-codec-1.10.jar
commons-collections-3.2.2.jar
commons-compiler-3.0.8.jar
-commons-compress-1.4.1.jar
+commons-compress-1.8.1.jar
commons-configuration2-2.1.1.jar
commons-crypto-1.0.0.jar
commons-daemon-1.0.13.jar
@@ -216,7 +216,7 @@ univocity-parsers-2.6.3.jar
validation-api-1.1.0.Final.jar
woodstox-core-5.0.3.jar
xbean-asm6-shaded-4.8.jar
-xz-1.0.jar
+xz-1.5.jar
zjsonpatch-0.3.0.jar
zookeeper-3.4.9.jar
zstd-jni-1.3.2-2.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9f60edc..be84661 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
<ivy.version>2.4.0</ivy.version>
<oro.version>2.0.8</oro.version>
<codahale.metrics.version>3.1.5</codahale.metrics.version>
- <avro.version>1.7.7</avro.version>
+ <avro.version>1.8.2</avro.version>
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
<jets3t.version>0.9.4</jets3t.version>
<aws.kinesis.client.version>1.7.3</aws.kinesis.client.version>
http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/sql/core/pom.xml
----------------------------------------------------------------------
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 8873b00..9cd6776 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -174,19 +174,6 @@
<artifactId>parquet-avro</artifactId>
<scope>test</scope>
</dependency>
- <!--
- This version of avro test-dep is different from the one defined
- in the parent pom. The parent pom has avro 1.7.7 test-dep for Hadoop.
- Here, ParquetAvroCompatibilitySuite uses parquet-avro's AvroParquetWriter
- which uses avro 1.8.0+ specific API. In Maven 3, we need to have
- this here to have different versions for the same artifact.
- -->
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>1.8.1</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org