You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2022/03/14 15:52:52 UTC
[flink] branch master updated: [FLINK-25927][connectors][build] Consistent flink-connector-base usage
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 9c77f13 [FLINK-25927][connectors][build] Consistent flink-connector-base usage
9c77f13 is described below
commit 9c77f13e587d7cb52469cac934c6aaec28dcd17d
Author: Alexander Fedulov <14...@users.noreply.github.com>
AuthorDate: Mon Mar 14 16:52:19 2022 +0100
[FLINK-25927][connectors][build] Consistent flink-connector-base usage
flink-connector-base was previously inconsistently used in connectors (directly shaded in some and transitively pulled in via flink-connector-files which was itself shaded in the table uber jar). FLINK-24687 moved flink-connector-files out from the flink-table uber jar. This commit implements a combined approach for ensuring a smooth transition for both Flink users and for external connector developers, as outlined in this:
- all internal Flink connectors that depend on flink-connector-base now shade and relocate it
- for compatibility, until external developers implement the same change, flink-connector-base is also included into flink-dist
---
.../flink-connector-aws-kinesis-firehose/pom.xml | 34 +++++++++---
.../flink-connector-aws-kinesis-streams/pom.xml | 34 +++++++++---
.../flink-connector-elasticsearch6/pom.xml | 27 ++++++++++
.../flink-connector-elasticsearch7/pom.xml | 27 ++++++++++
flink-connectors/flink-connector-files/pom.xml | 6 +++
flink-connectors/flink-connector-hive/pom.xml | 7 +++
flink-connectors/flink-connector-kafka/pom.xml | 27 ++++++++++
flink-connectors/flink-connector-pulsar/pom.xml | 26 +++++++++
.../pom.xml | 1 -
.../pom.xml | 1 -
.../flink-sql-connector-hbase-1.4/pom.xml | 4 ++
.../flink-sql-connector-hbase-2.2/pom.xml | 4 ++
flink-connectors/flink-sql-connector-kafka/pom.xml | 1 -
.../flink-sql-connector-kinesis/pom.xml | 1 -
flink-dist/pom.xml | 6 +++
.../pom.xml | 8 +++
.../pom.xml | 7 +++
.../flink-streaming-kinesis-test/pom.xml | 7 +++
flink-formats/flink-orc/pom.xml | 33 ++++++++++++
flink-formats/flink-parquet/pom.xml | 33 ++++++++++++
tools/ci/compile.sh | 4 ++
tools/ci/shade.sh | 61 +++++++++++++++++++++-
22 files changed, 342 insertions(+), 17 deletions(-)
diff --git a/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml b/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml
index 7463999..624b66e 100644
--- a/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml
+++ b/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml
@@ -48,12 +48,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-connector-aws-base</artifactId>
<version>${project.version}</version>
</dependency>
@@ -164,6 +158,34 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-aws-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.firehose.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml b/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml
index 7c864ef..5620a1b 100644
--- a/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml
+++ b/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml
@@ -48,12 +48,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-connector-aws-base</artifactId>
<version>${project.version}</version>
</dependency>
@@ -145,6 +139,34 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-aws-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.kinesis.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-connectors/flink-connector-elasticsearch6/pom.xml b/flink-connectors/flink-connector-elasticsearch6/pom.xml
index 6a5b5e8..92bc8f1 100644
--- a/flink-connectors/flink-connector-elasticsearch6/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch6/pom.xml
@@ -199,6 +199,33 @@ under the License.
<forkCount>1</forkCount>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-elasticsearch-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.elasticsearch6.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-connectors/flink-connector-elasticsearch7/pom.xml b/flink-connectors/flink-connector-elasticsearch7/pom.xml
index 1c1a195..4f01962 100644
--- a/flink-connectors/flink-connector-elasticsearch7/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch7/pom.xml
@@ -196,6 +196,33 @@ under the License.
<forkCount>1</forkCount>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-elasticsearch-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.elasticsearch7.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml
index 2a0ed9d..e49cc84 100644
--- a/flink-connectors/flink-connector-files/pom.xml
+++ b/flink-connectors/flink-connector-files/pom.xml
@@ -152,6 +152,12 @@ under the License.
<include>org.apache.flink:flink-connector-base</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.files.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
</configuration>
</execution>
</executions>
diff --git a/flink-connectors/flink-connector-hive/pom.xml b/flink-connectors/flink-connector-hive/pom.xml
index 28db2b0..94728ee 100644
--- a/flink-connectors/flink-connector-hive/pom.xml
+++ b/flink-connectors/flink-connector-hive/pom.xml
@@ -549,6 +549,13 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<!--flink-java and flink-clients test dependencies used for HiveInputFormatTest-->
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
index 3cacec0..96484a9 100644
--- a/flink-connectors/flink-connector-kafka/pom.xml
+++ b/flink-connectors/flink-connector-kafka/pom.xml
@@ -288,6 +288,33 @@ under the License.
<argLine>-Xms256m -Xmx2048m -Dmvn.forkNumber=${surefire.forkNumber} -XX:-UseGCOverheadLimit -Duser.country=US -Duser.language=en</argLine>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <shadeTestJar>true</shadeTestJar>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.kafka.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/flink-connectors/flink-connector-pulsar/pom.xml b/flink-connectors/flink-connector-pulsar/pom.xml
index 655c57f..53e5ca4 100644
--- a/flink-connectors/flink-connector-pulsar/pom.xml
+++ b/flink-connectors/flink-connector-pulsar/pom.xml
@@ -340,6 +340,32 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.pulsar.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml b/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml
index 8b4a2df..8bcdd19 100644
--- a/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml
+++ b/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml
@@ -54,7 +54,6 @@
<configuration>
<artifactSet>
<includes>
- <include>org.apache.flink:flink-connector-base</include>
<include>org.apache.flink:flink-connector-aws-base</include>
<include>org.apache.flink:flink-connector-aws-kinesis-firehose</include>
<include>software.amazon.awssdk:*</include>
diff --git a/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml b/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml
index 0804a3a..65fc447 100644
--- a/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml
+++ b/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml
@@ -55,7 +55,6 @@
<configuration>
<artifactSet>
<includes>
- <include>org.apache.flink:flink-connector-base</include>
<include>org.apache.flink:flink-connector-aws-base</include>
<include>org.apache.flink:flink-connector-aws-kinesis-streams</include>
<include>software.amazon.awssdk:*</include>
diff --git a/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml b/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml
index 78c1e5f..7cf5b8d 100644
--- a/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml
+++ b/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml
@@ -113,6 +113,10 @@ under the License.
</filter>
</filters>
<relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.sql.hbase14.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
<!-- Force relocation of all HBase dependencies. -->
<relocation>
<pattern>org.apache.zookeeper</pattern>
diff --git a/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml b/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml
index 2b2873d..9cfae8b 100644
--- a/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml
+++ b/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml
@@ -117,6 +117,10 @@ under the License.
</filter>
</filters>
<relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.sql.hbase22.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
<!-- Force relocation of all HBase dependencies. -->
<relocation>
<pattern>org.apache.zookeeper</pattern>
diff --git a/flink-connectors/flink-sql-connector-kafka/pom.xml b/flink-connectors/flink-sql-connector-kafka/pom.xml
index ad63e6c..256b9a0 100644
--- a/flink-connectors/flink-sql-connector-kafka/pom.xml
+++ b/flink-connectors/flink-sql-connector-kafka/pom.xml
@@ -58,7 +58,6 @@ under the License.
<configuration>
<artifactSet>
<includes>
- <include>org.apache.flink:flink-connector-base</include>
<include>org.apache.flink:flink-connector-kafka</include>
<include>org.apache.kafka:*</include>
</includes>
diff --git a/flink-connectors/flink-sql-connector-kinesis/pom.xml b/flink-connectors/flink-sql-connector-kinesis/pom.xml
index 1eb1694..d75a85f 100644
--- a/flink-connectors/flink-sql-connector-kinesis/pom.xml
+++ b/flink-connectors/flink-sql-connector-kinesis/pom.xml
@@ -58,7 +58,6 @@ under the License.
<configuration>
<artifactSet>
<includes>
- <include>org.apache.flink:flink-connector-base</include>
<include>org.apache.flink:flink-connector-kinesis</include>
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>com.fasterxml.jackson.core:jackson-databind</include>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 13b36ce..22bbb09 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -138,6 +138,12 @@ under the License.
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!-- Default file system support. The Hadoop dependency -->
<!-- is optional, so not being added to the dist jar -->
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml
index ed4a6c8..f32d6f1 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml
@@ -51,6 +51,14 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml
index 423afd5..d2aa132 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml
@@ -65,6 +65,13 @@
<artifactId>jackson-databind</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml b/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml
index fafa407..c6cefd8 100644
--- a/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml
+++ b/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml
@@ -89,6 +89,13 @@ under the License.
</dependency>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>compile</scope>
diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml
index fad8c27..bf5a380 100644
--- a/flink-formats/flink-orc/pom.xml
+++ b/flink-formats/flink-orc/pom.xml
@@ -45,6 +45,12 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!-- Table ecosystem and filesystem connector -->
<dependency>
@@ -204,6 +210,33 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.format.orc.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml
index f5e340a..8efa5a2 100644
--- a/flink-formats/flink-parquet/pom.xml
+++ b/flink-formats/flink-parquet/pom.xml
@@ -45,6 +45,12 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!-- Table ecosystem and filesystem connector -->
<dependency>
@@ -308,6 +314,33 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.format.parquet.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh
index e6013f9..c98fbae 100755
--- a/tools/ci/compile.sh
+++ b/tools/ci/compile.sh
@@ -108,6 +108,10 @@ check_shaded_artifacts_connector_elasticsearch 6
EXIT_CODE=$(($EXIT_CODE+$?))
check_shaded_artifacts_connector_elasticsearch 7
EXIT_CODE=$(($EXIT_CODE+$?))
+check_one_per_package_file_connector_base
+EXIT_CODE=$(($EXIT_CODE+$?))
+check_relocated_file_connector_base
+EXIT_CODE=$(($EXIT_CODE+$?))
echo "============ Run license check ============"
diff --git a/tools/ci/shade.sh b/tools/ci/shade.sh
index 922bba4..5931faf 100755
--- a/tools/ci/shade.sh
+++ b/tools/ci/shade.sh
@@ -153,7 +153,7 @@ check_shaded_artifacts_connector_elasticsearch() {
VARIANT=$1
find flink-connectors/flink-connector-elasticsearch${VARIANT}/target/flink-connector-elasticsearch${VARIANT}*.jar ! -name "*-tests.jar" -exec jar tf {} \; > allClasses
- UNSHADED_CLASSES=`cat allClasses | grep -v -e '^META-INF' -e '^assets' -e "^org/apache/flink/connector/elasticsearch/" -e "^org/apache/flink/streaming/connectors/elasticsearch/" -e "^org/apache/flink/streaming/connectors/elasticsearch${VARIANT}/" -e "^org/apache/flink/table/descriptors/" -e "^org/elasticsearch/" | grep '\.class$'`
+ UNSHADED_CLASSES=`cat allClasses | grep -v -e '^META-INF' -e '^assets' -e "^org/apache/flink/connector/elasticsearch/" -e "^org/apache/flink/streaming/connectors/elasticsearch/" -e "^org/apache/flink/streaming/connectors/elasticsearch${VARIANT}/" -e "^org/apache/flink/connector/elasticsearch${VARIANT}/shaded/" -e "^org/apache/flink/table/descriptors/" -e "^org/elasticsearch/" | grep '\.class$'`
if [ "$?" = "0" ]; then
echo "=============================================================================="
echo "Detected unshaded dependencies in flink-connector-elasticsearch${VARIANT}'s fat jar:"
@@ -173,3 +173,62 @@ check_shaded_artifacts_connector_elasticsearch() {
return 0
}
+
+check_one_per_package() {
+ read foo
+ if [ $foo -gt 1 ]
+ then
+ echo "ERROR - CHECK FAILED: $1 is shaded multiple times!"
+ exit 1
+ else
+ echo "OK"
+ fi
+}
+
+check_relocated() {
+ read foo
+ if [ $foo -ne 0 ]
+ then
+ echo "ERROR - CHECK FAILED: found $1 classes that where not relocated!"
+ exit 1
+ else
+ echo "OK"
+ fi
+}
+
+check_one_per_package_file_connector_base() {
+ echo "Checking that flink-connector-base is included only once:"
+ echo "__________________________________________________________________________"
+
+ CONNECTOR_JARS=$(find flink-connectors -type f -name '*.jar' | grep -vE "original|connector-hive" | grep -v '\-test');
+ EXIT_CODE=0
+
+ for i in $CONNECTOR_JARS;
+ do
+ echo -n "- $i: ";
+ jar tf $i | grep 'org/apache/flink/connector/base/source/reader/RecordEmitter' | wc -l | check_one_per_package "flink-connector-base";
+ EXIT_CODE=$((EXIT_CODE+$?))
+ done;
+ return $EXIT_CODE;
+}
+
+check_relocated_file_connector_base() {
+ echo -e "\n\n"
+ echo "Checking that flink-connector-base is relocated:"
+ echo "__________________________________________________________________________"
+
+ CONNECTOR_JARS=$(find flink-connectors -type f -name '*.jar' | \
+ grep -v original | grep -v '\-test' | grep -v 'flink-connectors/flink-connector-base');
+
+ EXIT_CODE=0
+ for i in $CONNECTOR_JARS;
+ do
+ echo -n "- $i: ";
+ jar tf $i | grep '^org/apache/flink/connector/base/source/reader/RecordEmitter' | wc -l | check_relocated "flink-connector-base";
+ EXIT_CODE=$((EXIT_CODE+$?))
+ done;
+ return $EXIT_CODE;
+}
+
+
+