You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/04/29 16:22:44 UTC
[iceberg] branch master updated: Spark: Depend on Parquet directly for Spark modules (#4667)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new f0a45893b Spark: Depend on Parquet directly for Spark modules (#4667)
f0a45893b is described below
commit f0a45893bc612c60758af26b8870e1d37a8e41db
Author: Steven Zhen Wu <st...@gmail.com>
AuthorDate: Fri Apr 29 09:22:38 2022 -0700
Spark: Depend on Parquet directly for Spark modules (#4667)
---
spark/v3.0/build.gradle | 5 +++++
spark/v3.1/build.gradle | 5 +++++
spark/v3.2/build.gradle | 5 +++++
3 files changed, 15 insertions(+)
diff --git a/spark/v3.0/build.gradle b/spark/v3.0/build.gradle
index d9db09374..3f68b929d 100644
--- a/spark/v3.0/build.gradle
+++ b/spark/v3.0/build.gradle
@@ -64,9 +64,13 @@ project(':iceberg-spark:iceberg-spark-3.0_2.12') {
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
exclude group: 'org.roaringbitmap'
}
+ implementation("org.apache.parquet:parquet-column")
+ implementation("org.apache.parquet:parquet-hadoop")
+
implementation("org.apache.orc:orc-core::nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
@@ -136,6 +140,7 @@ project(":iceberg-spark:iceberg-spark-extensions-3.0_2.12") {
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
exclude group: 'org.roaringbitmap'
}
diff --git a/spark/v3.1/build.gradle b/spark/v3.1/build.gradle
index 767126f64..ad8372544 100644
--- a/spark/v3.1/build.gradle
+++ b/spark/v3.1/build.gradle
@@ -64,9 +64,13 @@ project(':iceberg-spark:iceberg-spark-3.1_2.12') {
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
exclude group: 'org.roaringbitmap'
}
+ implementation("org.apache.parquet:parquet-column")
+ implementation("org.apache.parquet:parquet-hadoop")
+
implementation("org.apache.orc:orc-core::nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
@@ -136,6 +140,7 @@ project(":iceberg-spark:iceberg-spark-extensions-3.1_2.12") {
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
exclude group: 'org.roaringbitmap'
}
diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle
index 3e35cc44f..ae76a0f91 100644
--- a/spark/v3.2/build.gradle
+++ b/spark/v3.2/build.gradle
@@ -65,11 +65,15 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
// to make sure io.netty.buffer only comes from project(':iceberg-arrow')
exclude group: 'io.netty', module: 'netty-buffer'
exclude group: 'org.roaringbitmap'
}
+ implementation("org.apache.parquet:parquet-column")
+ implementation("org.apache.parquet:parquet-hadoop")
+
implementation("org.apache.orc:orc-core::nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
@@ -141,6 +145,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
// to make sure io.netty.buffer only comes from project(':iceberg-arrow')
exclude group: 'io.netty', module: 'netty-buffer'
exclude group: 'org.roaringbitmap'