You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by sh...@apache.org on 2021/09/11 01:51:18 UTC
[parquet-mr] branch parquet-1.12.x updated: PARQUET-2043: Fail for
undeclared dependencies (#916)
This is an automated email from the ASF dual-hosted git repository.
shangxinli pushed a commit to branch parquet-1.12.x
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/parquet-1.12.x by this push:
new f5595fe PARQUET-2043: Fail for undeclared dependencies (#916)
f5595fe is described below
commit f5595fedc36524bab58e0d36b34be26eaa4177e6
Author: Gabor Szadovszky <ga...@apache.org>
AuthorDate: Mon Aug 16 11:19:36 2021 +0200
PARQUET-2043: Fail for undeclared dependencies (#916)
The purpose of this change is to fail the build if some classes are
used from not direct dependencies. Only classes from direct
dependencies shall be used.
Also fixed some references that broke this rule.
---
parquet-arrow/pom.xml | 20 -------
parquet-avro/pom.xml | 24 ++++++---
parquet-benchmarks/pom.xml | 43 +++++++++++----
parquet-cli/pom.xml | 62 +++++++++++++++++++++-
parquet-column/pom.xml | 10 ++++
parquet-common/pom.xml | 6 ---
parquet-encoding/pom.xml | 6 +++
parquet-format-structures/pom.xml | 12 +----
parquet-hadoop-bundle/pom.xml | 15 ++++++
parquet-hadoop/pom.xml | 50 ++++++++++++++---
.../SchemaControlEncryptionTest.java | 15 +++---
parquet-jackson/pom.xml | 15 ++++++
parquet-pig-bundle/pom.xml | 15 ++++++
parquet-pig/pom.xml | 32 ++++++++---
parquet-protobuf/pom.xml | 16 +++++-
parquet-scala/pom.xml | 6 +++
parquet-thrift/pom.xml | 42 ++++++++++-----
pom.xml | 55 +++++++++++++++++++
18 files changed, 353 insertions(+), 91 deletions(-)
diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml
index b7a9481..3ff1091 100644
--- a/parquet-arrow/pom.xml
+++ b/parquet-arrow/pom.xml
@@ -43,26 +43,6 @@
<version>${arrow.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-memory</artifactId>
- <version>${arrow.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-format</artifactId>
- <version>${arrow.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-encoding</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${project.version}</version>
diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml
index e7b8e6b..3b851ea 100644
--- a/parquet-avro/pom.xml
+++ b/parquet-avro/pom.xml
@@ -45,7 +45,7 @@
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
- <artifactId>parquet-format-structures</artifactId>
+ <artifactId>parquet-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -54,6 +54,11 @@
<version>${avro.version}</version>
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>${fastutil.version}</version>
@@ -61,14 +66,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml
index ae16bfb..dd8c2ee 100644
--- a/parquet-benchmarks/pom.xml
+++ b/parquet-benchmarks/pom.xml
@@ -38,25 +38,23 @@
<dependencies>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.parquet</groupId>
- <artifactId>parquet-encoding</artifactId>
+ <artifactId>parquet-column</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
@@ -79,6 +77,11 @@
<artifactId>fastutil</artifactId>
<version>${fastutil.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
</dependencies>
<build>
@@ -124,6 +127,24 @@
</execution>
</executions>
</plugin>
+ <!--
+ - Skip analyzing dependencies in this module so we can pack everything (including hadoop classes) to an uber
+ - jar so benchmarks can be executed easily
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
index d886951..fb416d6 100644
--- a/parquet-cli/pom.xml
+++ b/parquet-cli/pom.xml
@@ -39,6 +39,26 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-format-structures</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-column</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
@@ -47,6 +67,7 @@
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>${zstd-jni.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -67,6 +88,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-jackson</artifactId>
<version>${project.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>${jackson.groupId}</groupId>
@@ -74,6 +96,16 @@
<version>${jackson-databind.version}</version>
</dependency>
<dependency>
+ <groupId>${jackson.groupId}</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${jackson.groupId}</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>${jcommander.version}</version>
@@ -93,7 +125,35 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${jsr305.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.1.3</version>
<scope>provided</scope>
</dependency>
</dependencies>
diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml
index cb0c185..d00a7ce 100644
--- a/parquet-column/pom.xml
+++ b/parquet-column/pom.xml
@@ -70,6 +70,16 @@
<artifactId>zero-allocation-hashing</artifactId>
<version>${net.openhft.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>audience-annotations</artifactId>
+ <version>${yetus.audience-annotations.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
<dependency>
<groupId>com.carrotsearch</groupId>
diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
index a368446..b0bc410 100644
--- a/parquet-common/pom.xml
+++ b/parquet-common/pom.xml
@@ -61,12 +61,6 @@
<version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.yetus</groupId>
- <artifactId>audience-annotations</artifactId>
- <version>0.12.0</version>
- </dependency>
</dependencies>
<build>
diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml
index f1f7d9a..1452a68 100644
--- a/parquet-encoding/pom.xml
+++ b/parquet-encoding/pom.xml
@@ -44,6 +44,12 @@
<dependency>
<groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
<scope>test</scope>
diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml
index 4fb990b..395a8cd 100644
--- a/parquet-format-structures/pom.xml
+++ b/parquet-format-structures/pom.xml
@@ -47,7 +47,7 @@
<executions>
<execution>
<id>unpack</id>
- <phase>generate-sources</phase>
+ <phase>initialize</phase>
<goals>
<goal>unpack</goal>
</goals>
@@ -154,20 +154,10 @@
<dependencies>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>${format.thrift.version}</version>
</dependency>
- <dependency>
- <groupId>javax.annotation</groupId>
- <artifactId>javax.annotation-api</artifactId>
- <version>1.3.2</version>
- </dependency>
</dependencies>
<profiles>
diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
index 4212ae0..7d408d8 100644
--- a/parquet-hadoop-bundle/pom.xml
+++ b/parquet-hadoop-bundle/pom.xml
@@ -84,6 +84,21 @@
</execution>
</executions>
</plugin>
+ <!-- Skip analyzing dependencies in this module so we can depend on modules without any code references -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
index c11c560..8c14b73 100644
--- a/parquet-hadoop/pom.xml
+++ b/parquet-hadoop/pom.xml
@@ -48,6 +48,11 @@
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
@@ -56,19 +61,29 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-jackson</artifactId>
<version>${project.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>${jackson.groupId}</groupId>
@@ -127,6 +142,28 @@
<version>4.6.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>zero-allocation-hashing</artifactId>
+ <version>${net.openhft.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>audience-annotations</artifactId>
+ <version>${yetus.audience-annotations.version}</version>
+ </dependency>
</dependencies>
<build>
@@ -159,6 +196,7 @@
<groupId>com.github.rdblue</groupId>
<artifactId>brotli-codec</artifactId>
<version>${brotli-codec.version}</version>
+ <scope>runtime</scope>
<optional>true</optional>
</dependency>
</dependencies>
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
index 17fda97..862ae67 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
@@ -19,8 +19,6 @@
package org.apache.parquet.crypto.propertiesfactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.column.ColumnDescriptor;
@@ -28,7 +26,6 @@ import org.apache.parquet.crypto.EncryptionPropertiesFactory;
import org.apache.parquet.crypto.ParquetCipher;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroup;
-import org.apache.parquet.format.EncryptionAlgorithm;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.api.WriteSupport;
@@ -39,6 +36,8 @@ import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
@@ -58,13 +57,13 @@ import static org.junit.Assert.assertEquals;
public class SchemaControlEncryptionTest {
- private final static Log LOG = LogFactory.getLog(SchemaControlEncryptionTest.class);
+ private final static Logger LOG = LoggerFactory.getLogger(SchemaControlEncryptionTest.class);
private final static int numRecord = 1000;
private Random rnd = new Random(5);
-
+
// In the test We use a map to tell WriteSupport which columns to be encrypted with what key. In real use cases, people
- // can find whatever easy way to do so basing on how do they get these information, for example people can choose to
- // store in HMS, or other metastore.
+ // can find whatever easy way to do so basing on how do they get these information, for example people can choose to
+ // store in HMS, or other metastore.
private Map<String, Map<String, Object>> cryptoMetadata = new HashMap<>();
private Map<String, Object[]> testData = new HashMap<>();
@@ -122,7 +121,7 @@ public class SchemaControlEncryptionTest {
encryptParquetFile(file, conf);
decryptParquetFileAndValid(file, conf);
}
-
+
private void markEncryptColumns() {
Map<String, Object> ageMetadata = new HashMap<>();
ageMetadata.put("columnKeyMetaData", "age_key_id");
diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml
index c12e247..16d6e4f 100644
--- a/parquet-jackson/pom.xml
+++ b/parquet-jackson/pom.xml
@@ -90,6 +90,21 @@
</execution>
</executions>
</plugin>
+ <!-- Skip analyzing dependencies in this module so we can depend on jackson without any code references -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml
index eede369..a10b4eb 100644
--- a/parquet-pig-bundle/pom.xml
+++ b/parquet-pig-bundle/pom.xml
@@ -74,6 +74,21 @@
</execution>
</executions>
</plugin>
+ <!-- Skip analyzing dependencies in this module so we can depend on parquet-pig without any code references -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml
index a725184..ef05c7b 100644
--- a/parquet-pig/pom.xml
+++ b/parquet-pig/pom.xml
@@ -48,10 +48,16 @@
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
- <artifactId>parquet-format-structures</artifactId>
+ <artifactId>parquet-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-jackson</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
@@ -65,14 +71,16 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-jackson</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>${jackson.groupId}</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>${jackson.groupId}</groupId>
@@ -80,6 +88,11 @@
<version>${jackson-databind.version}</version>
</dependency>
<dependency>
+ <groupId>${jackson.groupId}</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${project.version}</version>
@@ -110,6 +123,11 @@
<version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml
index 990d5e9..2e82b49 100644
--- a/parquet-protobuf/pom.xml
+++ b/parquet-protobuf/pom.xml
@@ -87,10 +87,24 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml
index 5f5f759..9786b10 100644
--- a/parquet-scala/pom.xml
+++ b/parquet-scala/pom.xml
@@ -65,6 +65,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.scalactic</groupId>
+ <artifactId>scalactic_${scala.binary.version}</artifactId>
+ <version>3.0.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml
index 51e4aa3..f0d8e77 100644
--- a/parquet-thrift/pom.xml
+++ b/parquet-thrift/pom.xml
@@ -48,16 +48,24 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.twitter.elephantbird</groupId>
@@ -87,16 +95,22 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-jackson</artifactId>
<version>${project.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>${jackson.groupId}</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
</dependency>
<dependency>
<groupId>${jackson.groupId}</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson-databind.version}</version>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -137,6 +151,11 @@
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
<scope>test</scope>
@@ -155,7 +174,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
</dependencies>
<dependencyManagement>
diff --git a/pom.xml b/pom.xml
index 9f60ac9..f28f7d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,12 +106,14 @@
<net.openhft.version>0.9</net.openhft.version>
<maven-javadoc-plugin.version>3.1.0</maven-javadoc-plugin.version>
<exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
+ <yetus.audience-annotations.version>0.13.0</yetus.audience-annotations.version>
<!-- parquet-cli dependencies -->
<opencsv.version>2.3</opencsv.version>
<jcommander.version>1.72</jcommander.version>
<zstd-jni.version>1.4.9-1</zstd-jni.version>
<commons-text.version>1.8</commons-text.version>
+ <jsr305.version>3.0.2</jsr305.version>
<!-- properties for the profiles -->
<surefire.logLevel>INFO</surefire.logLevel>
@@ -156,6 +158,44 @@
</dependency>
</dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<reporting>
<plugins>
<plugin>
@@ -521,6 +561,21 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <failOnWarning>true</failOnWarning>
+ <ignoreNonCompile>true</ignoreNonCompile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>