You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2019/08/02 13:19:40 UTC
[flink] 03/03: [FLINK-13504][table] Fixed shading issues in table
modules.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3fa4898a4d1b2d69ef64f48b5a5aefafa2e03974
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 1 11:00:14 2019 +0200
[FLINK-13504][table] Fixed shading issues in table modules.
Properly include the flink-sql-parser into flink-planner* modules shaded
jars. Removed the flink-sql-parser module from flink-table-uber* jars as
it is already included in the flink-planner* modules.
This closes #9313.
---
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../test-scripts/test_table_shaded_dependencies.sh | 111 +++++++++++++++++++++
flink-table/flink-table-planner-blink/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 2 +-
flink-table/flink-table-uber-blink/pom.xml | 2 +-
flink-table/flink-table-uber/pom.xml | 3 +-
6 files changed, 117 insertions(+), 5 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index d50b328..a97b6a0 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -158,6 +158,8 @@ run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_h
run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25"
+run_test "Dependency shading of table modules test" "$END_TO_END_DIR/test-scripts/test_table_shaded_dependencies.sh"
+
################################################################################
# Sticky Scheduling
################################################################################
diff --git a/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
new file mode 100755
index 0000000..cdbec5f
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+FLINK_SCALA_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target" | sed -n "s/.*flink-table-runtime-blink_\(.*\)-tests\.jar/\1/p" | uniq`
+FLINK_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target" | sed -n "s/.*flink-table-api-java-\(.*\)-tests\.jar/\1/p" | uniq`
+
+# This checks the bytecode for dependencies on external classes. Some classes below
+# are not available in the uber jar. We check for classes that we expect in the uber jar with
+# checkAllowedPackages method.
+function checkCodeDependencies {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jdeps $JAR |\
+ grep "^\s*\->" |\
+ `# jdk dependencies` \
+ grep -v "^\s*\-> java." |\
+ grep -v "^\s*\-> sun.misc." |\
+ grep -v "^\s*\-> javax." |\
+ `# scala dependencies` \
+ grep -v "^\s*\-> scala" |\
+ `# flink dependencies` \
+ grep -v "^\s*\-> org.apache.flink" |\
+ `# janino dependencies` \
+ grep -v "^\s*\-> org.codehaus.janino" |\
+ grep -v "^\s*\-> org.codehaus.commons" |\
+ grep -v "^\s*\-> org.apache.tools.ant" |\
+ `# calcite dependencies` \
+ grep -v "^\s*\-> org.apache.calcite" |\
+ grep -v "^\s*\-> org.pentaho.aggdes" |\
+ grep -v "^\s*\-> org.apache.commons.lang3" |\
+ grep -v "^\s*\-> org.apache.commons.math3" |\
+ grep -v "^\s*\-> org.apache.commons.dbcp2" |\
+ grep -v "^\s*\-> org.apache.http" |\
+ grep -v "^\s*\-> org.w3c.dom" |\
+ grep -v "^\s*\-> org.xml.sax" |\
+ grep -v "^\s*\-> org.ietf.jgss" |\
+ grep -v "^\s*\-> com.esri.core." |\
+ grep -v "^\s*\-> com.yahoo.sketches.hll." |\
+ grep -v "^\s*\-> org.slf4j" |\
+ grep -v "^\s*\-> org.json" |\
+ grep -v "^\s*\-> org.apache.tapestry5.json." |\
+ grep -v "^\s*\-> org.codehaus.jettison" |\
+ grep -v "^\s*\-> net.minidev.json" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted dependencies in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted dependencies in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+# Checks that the uber jars contain only flink, relocated packages, or packages that we
+# consciously decided to include as not relocated.
+function checkAllowedPackages {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jar tf $JAR |\
+ grep ".*class" |\
+ grep -v "org/codehaus/janino" |\
+ grep -v "org/codehaus/commons" |\
+ grep -v "org/apache/calcite" |\
+ grep -v "org/apache/flink" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted classes in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted classes in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
diff --git a/flink-table/flink-table-planner-blink/pom.xml b/flink-table/flink-table-planner-blink/pom.xml
index bd6ce79..f9e7f2b 100644
--- a/flink-table/flink-table-planner-blink/pom.xml
+++ b/flink-table/flink-table-planner-blink/pom.xml
@@ -336,7 +336,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner-blink dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<!-- flink-table-runtime-blink dependencies -->
<include>org.codehaus.janino:*</include>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index d4c00d0..9ab8f19 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -339,7 +339,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<include>org.codehaus.janino:*</include>
<include>joda-time:*</include>
<include>joda-convert:*</include>
diff --git a/flink-table/flink-table-uber-blink/pom.xml b/flink-table/flink-table-uber-blink/pom.xml
index ffd40a4..32517ba 100644
--- a/flink-table/flink-table-uber-blink/pom.xml
+++ b/flink-table/flink-table-uber-blink/pom.xml
@@ -102,8 +102,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>
diff --git a/flink-table/flink-table-uber/pom.xml b/flink-table/flink-table-uber/pom.xml
index aa2c434..394417d 100644
--- a/flink-table/flink-table-uber/pom.xml
+++ b/flink-table/flink-table-uber/pom.xml
@@ -86,7 +86,6 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
- <!-- Exclude all flink-dist files and only include flink-table-* -->
<execution>
<id>shade-flink</id>
<phase>package</phase>
@@ -96,8 +95,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>