You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2023/04/07 09:50:51 UTC
[doris-shade] branch master updated: * Add Haddop 2 shade package and rename guava and gson package (#8)
This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-shade.git
The following commit(s) were added to refs/heads/master by this push:
new 66afe29 * Add Haddop 2 shade package and rename guava and gson package (#8)
66afe29 is described below
commit 66afe29240113cd59a17a5289ee2b2dad42e7e16
Author: Calvin Kirs <ki...@apache.org>
AuthorDate: Fri Apr 7 17:50:46 2023 +0800
* Add Haddop 2 shade package and rename guava and gson package (#8)
* * Add Haddop 2 shade package and rename guava and gson package
* Add hive3 shade package and rename guava and gson package
* Add Spring5 shade package
* Add Hudi-MR 0.13 shade package and rename guava and gson package
* Add Thrift 13 shade package
* Add license header
---
{hudi-mr-shade-0.13 => hadoop-aws-shade-2}/pom.xml | 16 ++-
{hadoop-shade-2 => hadoop-cloud-shade-2}/pom.xml | 111 +++++++++++----------
hadoop-shade-2/pom.xml | 99 +-----------------
hive-shade-3/pom.xml | 98 ++++++++++++++++++
hudi-mr-shade-0.13/pom.xml | 70 +++++++++++++
pom.xml | 2 +
6 files changed, 240 insertions(+), 156 deletions(-)
diff --git a/hudi-mr-shade-0.13/pom.xml b/hadoop-aws-shade-2/pom.xml
similarity index 84%
copy from hudi-mr-shade-0.13/pom.xml
copy to hadoop-aws-shade-2/pom.xml
index f7258d2..ef75270 100644
--- a/hudi-mr-shade-0.13/pom.xml
+++ b/hadoop-aws-shade-2/pom.xml
@@ -25,17 +25,17 @@ under the License.
<version>1.0.1-SNAPSHOT</version>
</parent>
- <artifactId>hudi-mr-shade-0.13</artifactId>
+ <artifactId>hadoop-aws-shade-2</artifactId>
<properties>
+ <hadoop.version>2.10.2</hadoop.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <hudi.version>0.13.0</hudi.version>
</properties>
<dependencies>
<dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-hadoop-mr</artifactId>
- <version>${hudi.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
@@ -65,11 +65,7 @@ under the License.
<relocations>
<relocation>
<pattern>com.google.common</pattern>
- <shadedPattern>shade.doris.hudi.mr.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.gson</pattern>
- <shadedPattern>shade.doris.hudi.mr.com.google.gson</shadedPattern>
+ <shadedPattern>shade.doris.hadoop.com.google.common</shadedPattern>
</relocation>
</relocations>
</configuration>
diff --git a/hadoop-shade-2/pom.xml b/hadoop-cloud-shade-2/pom.xml
similarity index 74%
copy from hadoop-shade-2/pom.xml
copy to hadoop-cloud-shade-2/pom.xml
index f8c737c..8ec2798 100644
--- a/hadoop-shade-2/pom.xml
+++ b/hadoop-cloud-shade-2/pom.xml
@@ -25,28 +25,32 @@ under the License.
<version>1.0.1-SNAPSHOT</version>
</parent>
- <artifactId>hadoop-shade-2</artifactId>
+ <artifactId>hadoop-cloud-shade-2</artifactId>
<properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.10.2</hadoop.version>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
+
+
<dependencies>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-huaweicloud -->
+ <!-- https://mvnrepository.com/artifact/com.huaweicloud/esdk-obs-java-bundle -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-huaweicloud</artifactId>
- <version>2.8.3</version>
+ <groupId>com.huaweicloud</groupId>
+ <artifactId>esdk-obs-java-bundle</artifactId>
+ <version>3.21.8</version>
<exclusions>
<exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>org.apache.logging.log4j</artifactId>
+ <groupId>log4j-core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>org.apache.logging.log4j</artifactId>
+ <groupId>log4j-api</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>stax</artifactId>
+ <groupId>stax-api</groupId>
</exclusion>
</exclusions>
</dependency>
@@ -55,52 +59,49 @@ under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aliyun</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
-
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-auth -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
+ <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-huaweicloud -->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-huaweicloud</artifactId>
+ <version>2.8.3</version>
<exclusions>
- <exclusion>
- <artifactId>netty-all</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
-
- <!-- https://mvnrepository.com/artifact/com.huaweicloud/esdk-obs-java-bundle -->
- <dependency>
- <groupId>com.huaweicloud</groupId>
- <artifactId>esdk-obs-java-bundle</artifactId>
- <version>3.21.8</version>
- </dependency>
-
-
- <!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
<dependency>
<groupId>com.qcloud.cos</groupId>
<artifactId>hadoop-cos</artifactId>
@@ -190,7 +191,15 @@ under the License.
<relocations>
<relocation>
<pattern>com.google.common</pattern>
- <shadedPattern>shade.doris.hdfs.com.google.common</shadedPattern>
+ <shadedPattern>shade.doris.hadoop.cloud.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>shade.doris.hadoop.cloud.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>shade.doris.hadoop.io.netty</shadedPattern>
</relocation>
</relocations>
</configuration>
diff --git a/hadoop-shade-2/pom.xml b/hadoop-shade-2/pom.xml
index f8c737c..f09d6d2 100644
--- a/hadoop-shade-2/pom.xml
+++ b/hadoop-shade-2/pom.xml
@@ -32,31 +32,6 @@ under the License.
<hadoop.version>2.10.2</hadoop.version>
</properties>
<dependencies>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-huaweicloud -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-huaweicloud</artifactId>
- <version>2.8.3</version>
- <exclusions>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aliyun -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -92,77 +67,7 @@ under the License.
</exclusions>
</dependency>
- <!-- https://mvnrepository.com/artifact/com.huaweicloud/esdk-obs-java-bundle -->
- <dependency>
- <groupId>com.huaweicloud</groupId>
- <artifactId>esdk-obs-java-bundle</artifactId>
- <version>3.21.8</version>
- </dependency>
-
-
- <!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
- <dependency>
- <groupId>com.qcloud.cos</groupId>
- <artifactId>hadoop-cos</artifactId>
- <version>2.8.5-8.1.8</version>
- </dependency>
</dependencies>
- <profiles>
- <!-- for custom internal repository -->
- <profile>
- <id>custom-env</id>
- <activation>
- <property>
- <name>env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <repository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </pluginRepository>
- </pluginRepositories>
- </profile>
- <!-- for general repository -->
- <profile>
- <id>general-env</id>
- <activation>
- <property>
- <name>!env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <!-- for java-cup -->
- <repository>
- <id>cloudera-thirdparty</id>
- <url>https://repository.cloudera.com/content/repositories/third-party/</url>
- </repository>
- <!-- for bdb je -->
- <repository>
- <id>oracleReleases</id>
- <url>http://download.oracle.com/maven</url>
- </repository>
- <!-- for huawei obs sdk -->
- <repository>
- <id>huawei-obs-sdk</id>
- <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <!-- for cup-maven-plugin -->
- <pluginRepository>
- <id>spring-plugins</id>
- <url>http://repo.spring.io/plugins-release/</url>
- </pluginRepository>
- </pluginRepositories>
- </profile>
- </profiles>
<build>
<plugins>
<plugin>
@@ -192,6 +97,10 @@ under the License.
<pattern>com.google.common</pattern>
<shadedPattern>shade.doris.hdfs.com.google.common</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>shade.doris.hadoop.io.netty</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/hive-shade-3/pom.xml b/hive-shade-3/pom.xml
index 5828dc4..416d8e2 100644
--- a/hive-shade-3/pom.xml
+++ b/hive-shade-3/pom.xml
@@ -58,6 +58,22 @@ under the License.
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hive/hive-storage-api -->
@@ -101,6 +117,22 @@ under the License.
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-pool</groupId>
+ <artifactId>commons-pool</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -113,6 +145,14 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
@@ -159,6 +199,60 @@ under the License.
<groupId>org.apache.hive</groupId>
<artifactId>hive-standalone-metastore</artifactId>
<version>3.1.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
@@ -208,6 +302,10 @@ under the License.
<pattern>com.google.gson</pattern>
<shadedPattern>shade.doris.hive.com.google.gson</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>shade.doris.hive.io.netty</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/hudi-mr-shade-0.13/pom.xml b/hudi-mr-shade-0.13/pom.xml
index f7258d2..258715e 100644
--- a/hudi-mr-shade-0.13/pom.xml
+++ b/hudi-mr-shade-0.13/pom.xml
@@ -36,6 +36,72 @@ under the License.
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-hadoop-mr</artifactId>
<version>${hudi.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>fluent-hc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
<build>
@@ -71,6 +137,10 @@ under the License.
<pattern>com.google.gson</pattern>
<shadedPattern>shade.doris.hudi.mr.com.google.gson</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>shade.doris.hudi.mr.com.fasterxml.jackson</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/pom.xml b/pom.xml
index 8539309..dd8658d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,6 +49,8 @@ under the License.
<module>hudi-mr-shade-0.13</module>
<module>hive-shade-3</module>
<module>spring-shade-5</module>
+ <module>hadoop-cloud-shade-2</module>
+ <module>hadoop-aws-shade-2</module>
</modules>
<properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org