You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/06/22 02:10:27 UTC
[incubator-seatunnel] branch api-draft updated: [API-Draft] Fix hadoop shade can't be imported problem (#2045)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch api-draft
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/api-draft by this push:
new 24b4d624e [API-Draft] Fix hadoop shade can't be imported problem (#2045)
24b4d624e is described below
commit 24b4d624efa8eeced889add1fc355f59a0f90022
Author: Hisoka <fa...@qq.com>
AuthorDate: Wed Jun 22 10:10:22 2022 +0800
[API-Draft] Fix hadoop shade can't be imported problem (#2045)
* fix hadoop shade can't be imported problem
* fix hadoop shade can't be imported problem
* fix hadoop shade can't be imported problem
* remove useless known-dependencies
---
.github/workflows/backend.yml | 4 +--
.../seatunnel-connectors-seatunnel-dist/pom.xml | 30 +++++++++++++++----
.../seatunnel-connectors-seatunnel/pom.xml | 22 +++++++++++++-
.../seatunnel-hadoop2-shade/pom.xml | 34 ++++++++++++++++++++++
.../seatunnel-hadoop3-shade/pom.xml | 33 +++++++++++++++++++++
.../seatunnel-connector-seatunnel-hive/pom.xml | 5 ++--
.../hive/sink/HiveAggregatedCommitInfo.java | 3 +-
.../connectors/seatunnel/hive/sink/HiveSink.java | 12 ++++++++
.../hive/sink/file/writer/HdfsTxtFileWriter.java | 3 +-
.../seatunnel/hive/sink/file/writer/HdfsUtils.java | 11 +++----
tools/dependencies/known-dependencies-jdk11.txt | 1 -
11 files changed, 140 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index c4d2d8a5e..77ed47668 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -35,7 +35,7 @@ jobs:
strategy:
matrix:
java: ['8', '11']
- timeout-minutes: 120
+ timeout-minutes: 80
env:
MAVEN_OPTS: -Xmx2G -Xms2G
steps:
@@ -80,7 +80,7 @@ jobs:
strategy:
matrix:
java: ['8']
- timeout-minutes: 120
+ timeout-minutes: 80
env:
MAVEN_OPTS: -Xmx2G -Xms2G
steps:
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel-dist/pom.xml b/seatunnel-connectors/seatunnel-connectors-seatunnel-dist/pom.xml
index 5c777fcee..fcc98fb82 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel-dist/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel-dist/pom.xml
@@ -50,11 +50,6 @@
<artifactId>seatunnel-connector-seatunnel-http</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-connector-seatunnel-hive</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-connector-seatunnel-jdbc</artifactId>
@@ -77,6 +72,31 @@
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>jdk8</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <jdk>1.8</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-connector-seatunnel-hive</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>jdk11</id>
+ <activation>
+ <jdk>11</jdk>
+ </activation>
+ <modules>
+ </modules>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/pom.xml b/seatunnel-connectors/seatunnel-connectors-seatunnel/pom.xml
index 4ad764d18..50c705fe3 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/pom.xml
@@ -31,7 +31,6 @@
<artifactId>seatunnel-connectors-seatunnel</artifactId>
<modules>
- <module>seatunnel-connector-seatunnel-hive</module>
<module>seatunnel-connector-seatunnel-console</module>
<module>seatunnel-connector-seatunnel-fake</module>
<module>seatunnel-connector-seatunnel-kafka</module>
@@ -42,4 +41,25 @@
<module>seatunnel-connector-seatunnel-pulsar</module>
<module>seatunnel-connector-hadoop-shade</module>
</modules>
+
+ <profiles>
+ <profile>
+ <id>jdk8</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <jdk>1.8</jdk>
+ </activation>
+ <modules>
+ <module>seatunnel-connector-seatunnel-hive</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>jdk11</id>
+ <activation>
+ <jdk>11</jdk>
+ </activation>
+ <modules>
+ </modules>
+ </profile>
+ </profiles>
</project>
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop2-shade/pom.xml b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop2-shade/pom.xml
index dc3db0174..bc25d0c9c 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop2-shade/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop2-shade/pom.xml
@@ -48,11 +48,19 @@
</dependencies>
<build>
+
+ <finalName>${project.artifactId}-${project.version}</finalName>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
+ <minimizeJar>true</minimizeJar>
+ <createSourcesJar>true</createSourcesJar>
+ <shadeSourcesContent>true</shadeSourcesContent>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>org.apache.hadoop</artifact>
@@ -80,6 +88,10 @@
<shadedPattern>${seatunnel.shade.package}.org.apache.hadoop</shadedPattern>
</relocation>
</relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer" />
+ </transformers>
</configuration>
<executions>
<execution>
@@ -89,7 +101,29 @@
</goals>
</execution>
</executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>${basedir}/target/${project.artifactId}-${project.version}.jar</file>
+ <type>jar</type>
+ <classifier>optional</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop3-shade/pom.xml b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop3-shade/pom.xml
index 228fba036..4b5f0c67c 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop3-shade/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-hadoop-shade/seatunnel-hadoop3-shade/pom.xml
@@ -48,11 +48,18 @@
</dependencies>
<build>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
+ <minimizeJar>true</minimizeJar>
+ <createSourcesJar>true</createSourcesJar>
+ <shadeSourcesContent>true</shadeSourcesContent>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>org.apache.hadoop</artifact>
@@ -80,6 +87,10 @@
<shadedPattern>${seatunnel.shade.package}.org.apache.hadoop</shadedPattern>
</relocation>
</relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer" />
+ </transformers>
</configuration>
<executions>
<execution>
@@ -89,7 +100,29 @@
</goals>
</execution>
</executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>${basedir}/target/${project.artifactId}-${project.version}.jar</file>
+ <type>jar</type>
+ <classifier>optional</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/pom.xml b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/pom.xml
index 30745c996..a115aa64b 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/pom.xml
@@ -37,8 +37,9 @@
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop2-shade</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveAggregatedCommitInfo.java b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveAggregatedCommitInfo.java
index f1eee53de..025fbefbf 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveAggregatedCommitInfo.java
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveAggregatedCommitInfo.java
@@ -20,11 +20,12 @@ package org.apache.seatunnel.connectors.seatunnel.hive.sink;
import lombok.AllArgsConstructor;
import lombok.Data;
+import java.io.Serializable;
import java.util.Map;
@Data
@AllArgsConstructor
-public class HiveAggregatedCommitInfo {
+public class HiveAggregatedCommitInfo implements Serializable {
/**
* Storage the commit info in map.
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
index c55ebcd5e..0eef08992 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
@@ -19,6 +19,8 @@ package org.apache.seatunnel.connectors.seatunnel.hive.sink;
import org.apache.seatunnel.api.common.PrepareFailException;
import org.apache.seatunnel.api.common.SeaTunnelContext;
+import org.apache.seatunnel.api.serialization.DefaultSerializer;
+import org.apache.seatunnel.api.serialization.Serializer;
import org.apache.seatunnel.api.sink.SeaTunnelSink;
import org.apache.seatunnel.api.sink.SinkAggregatedCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
@@ -80,8 +82,18 @@ public class HiveSink implements SeaTunnelSink<SeaTunnelRow, HiveSinkState, Hive
}
+ @Override
+ public Optional<Serializer<HiveCommitInfo>> getCommitInfoSerializer() {
+ return Optional.of(new DefaultSerializer<>());
+ }
+
@Override
public Optional<SinkAggregatedCommitter<HiveCommitInfo, HiveAggregatedCommitInfo>> createAggregatedCommitter() throws IOException {
return Optional.of(new HiveSinkAggregatedCommitter());
}
+
+ @Override
+ public Optional<Serializer<HiveAggregatedCommitInfo>> getAggregatedCommitInfoSerializer() {
+ return Optional.of(new DefaultSerializer<>());
+ }
}
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsTxtFileWriter.java b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsTxtFileWriter.java
index 71b26568f..7a472f4cb 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsTxtFileWriter.java
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsTxtFileWriter.java
@@ -21,9 +21,10 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.hive.sink.HiveSinkConfig;
+import org.apache.seatunnel.shade.org.apache.hadoop.fs.FSDataOutputStream;
+
import lombok.Lombok;
import lombok.NonNull;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsUtils.java b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsUtils.java
index 23b1e5843..acfb845ab 100644
--- a/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsUtils.java
+++ b/seatunnel-connectors/seatunnel-connectors-seatunnel/seatunnel-connector-seatunnel-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/file/writer/HdfsUtils.java
@@ -17,11 +17,12 @@
package org.apache.seatunnel.connectors.seatunnel.hive.sink.file.writer;
+import org.apache.seatunnel.shade.org.apache.hadoop.conf.Configuration;
+import org.apache.seatunnel.shade.org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.seatunnel.shade.org.apache.hadoop.fs.FileSystem;
+import org.apache.seatunnel.shade.org.apache.hadoop.fs.Path;
+
import lombok.NonNull;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +36,7 @@ public class HdfsUtils {
public static FileSystem getHdfsFs(@NonNull String path)
throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ conf.set("fs.hdfs.impl", "org.apache.seatunnel.shade.org.apache.hadoop.hdfs.DistributedFileSystem");
conf.set("fs.defaultFs", path);
return FileSystem.get(conf);
}
diff --git a/tools/dependencies/known-dependencies-jdk11.txt b/tools/dependencies/known-dependencies-jdk11.txt
index 0397ec80c..b529d97ff 100644
--- a/tools/dependencies/known-dependencies-jdk11.txt
+++ b/tools/dependencies/known-dependencies-jdk11.txt
@@ -676,7 +676,6 @@ woodstox-core-5.0.3.jar
xbean-asm6-shaded-4.10.jar
xbean-asm6-shaded-4.8.jar
xercesImpl-2.9.1.jar
-xml-apis-1.3.04.jar
xmlbeans-3.1.0.jar
xmlenc-0.52.jar
xz-1.0.jar