You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2019/08/02 13:34:56 UTC
[flink] branch release-1.9 updated (9e4de1e -> 854d9e1)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 9e4de1e [FLINK-13544][connectors] Set parallelism of table sink operator to input transformation parallelism
new 7c202b0 [hotfix][table-sql-parser] Removed unnecessary dependency on guava.
new d10e332 [hotfix][table-planner] Added joda.convert to planner jar
new 854d9e1 [FLINK-13504][table] Fixed shading issues in table modules.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../test-scripts/test_table_shaded_dependencies.sh | 111 +++++++++++++++++++++
flink-table/flink-sql-parser/pom.xml | 5 -
.../apache/flink/sql/parser/ddl/SqlCreateView.java | 5 +-
flink-table/flink-table-planner-blink/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 7 +-
flink-table/flink-table-uber-blink/pom.xml | 2 +-
flink-table/flink-table-uber/pom.xml | 3 +-
8 files changed, 122 insertions(+), 15 deletions(-)
create mode 100755 flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
[flink] 03/03: [FLINK-13504][table] Fixed shading issues in table
modules.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 854d9e19a0e8e2b124418f0dfdc9bf7a2e70830c
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 1 11:00:14 2019 +0200
[FLINK-13504][table] Fixed shading issues in table modules.
Properly include the flink-sql-parser into flink-planner* modules shaded
jars. Removed the flink-sql-parser module from flink-table-uber* jars as
it is already included in the flink-planner* modules.
This closes #9313.
---
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../test-scripts/test_table_shaded_dependencies.sh | 111 +++++++++++++++++++++
flink-table/flink-table-planner-blink/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 2 +-
flink-table/flink-table-uber-blink/pom.xml | 2 +-
flink-table/flink-table-uber/pom.xml | 3 +-
6 files changed, 117 insertions(+), 5 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index afbd98f..3673154 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -155,6 +155,8 @@ run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_h
run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25"
+run_test "Dependency shading of table modules test" "$END_TO_END_DIR/test-scripts/test_table_shaded_dependencies.sh"
+
################################################################################
# Sticky Scheduling
################################################################################
diff --git a/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
new file mode 100755
index 0000000..cdbec5f
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+FLINK_SCALA_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target" | sed -n "s/.*flink-table-runtime-blink_\(.*\)-tests\.jar/\1/p" | uniq`
+FLINK_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target" | sed -n "s/.*flink-table-api-java-\(.*\)-tests\.jar/\1/p" | uniq`
+
+# This checks the bytecode for dependencies on external classes. Some classes below
+# are not available in the uber jar. We check for classes that we expect in the uber jar with
+# checkAllowedPackages method.
+function checkCodeDependencies {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jdeps $JAR |\
+ grep "^\s*\->" |\
+ `# jdk dependencies` \
+ grep -v "^\s*\-> java." |\
+ grep -v "^\s*\-> sun.misc." |\
+ grep -v "^\s*\-> javax." |\
+ `# scala dependencies` \
+ grep -v "^\s*\-> scala" |\
+ `# flink dependencies` \
+ grep -v "^\s*\-> org.apache.flink" |\
+ `# janino dependencies` \
+ grep -v "^\s*\-> org.codehaus.janino" |\
+ grep -v "^\s*\-> org.codehaus.commons" |\
+ grep -v "^\s*\-> org.apache.tools.ant" |\
+ `# calcite dependencies` \
+ grep -v "^\s*\-> org.apache.calcite" |\
+ grep -v "^\s*\-> org.pentaho.aggdes" |\
+ grep -v "^\s*\-> org.apache.commons.lang3" |\
+ grep -v "^\s*\-> org.apache.commons.math3" |\
+ grep -v "^\s*\-> org.apache.commons.dbcp2" |\
+ grep -v "^\s*\-> org.apache.http" |\
+ grep -v "^\s*\-> org.w3c.dom" |\
+ grep -v "^\s*\-> org.xml.sax" |\
+ grep -v "^\s*\-> org.ietf.jgss" |\
+ grep -v "^\s*\-> com.esri.core." |\
+ grep -v "^\s*\-> com.yahoo.sketches.hll." |\
+ grep -v "^\s*\-> org.slf4j" |\
+ grep -v "^\s*\-> org.json" |\
+ grep -v "^\s*\-> org.apache.tapestry5.json." |\
+ grep -v "^\s*\-> org.codehaus.jettison" |\
+ grep -v "^\s*\-> net.minidev.json" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted dependencies in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted dependencies in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+# Checks that the uber jars contain only flink, relocated packages, or packages that we
+# consciously decided to include as not relocated.
+function checkAllowedPackages {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jar tf $JAR |\
+ grep ".*class" |\
+ grep -v "org/codehaus/janino" |\
+ grep -v "org/codehaus/commons" |\
+ grep -v "org/apache/calcite" |\
+ grep -v "org/apache/flink" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted classes in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted classes in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
diff --git a/flink-table/flink-table-planner-blink/pom.xml b/flink-table/flink-table-planner-blink/pom.xml
index c3aa1ef..5325c64 100644
--- a/flink-table/flink-table-planner-blink/pom.xml
+++ b/flink-table/flink-table-planner-blink/pom.xml
@@ -336,7 +336,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner-blink dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<!-- flink-table-runtime-blink dependencies -->
<include>org.codehaus.janino:*</include>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index b46e182..ea4e4fc 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -339,7 +339,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<include>org.codehaus.janino:*</include>
<include>joda-time:*</include>
<include>joda-convert:*</include>
diff --git a/flink-table/flink-table-uber-blink/pom.xml b/flink-table/flink-table-uber-blink/pom.xml
index 8c12898..1b91b9a 100644
--- a/flink-table/flink-table-uber-blink/pom.xml
+++ b/flink-table/flink-table-uber-blink/pom.xml
@@ -102,8 +102,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>
diff --git a/flink-table/flink-table-uber/pom.xml b/flink-table/flink-table-uber/pom.xml
index a2e5d15..de4efa0 100644
--- a/flink-table/flink-table-uber/pom.xml
+++ b/flink-table/flink-table-uber/pom.xml
@@ -86,7 +86,6 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
- <!-- Exclude all flink-dist files and only include flink-table-* -->
<execution>
<id>shade-flink</id>
<phase>package</phase>
@@ -96,8 +95,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>
[flink] 02/03: [hotfix][table-planner] Added joda.convert to
planner jar
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d10e3321ba9318e6af8723399dadf5816a85d3a0
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Fri Aug 2 09:56:48 2019 +0200
[hotfix][table-planner] Added joda.convert to planner jar
---
flink-table/flink-table-planner/pom.xml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index 1976773..b46e182 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -342,6 +342,7 @@ under the License.
<include>org.apache.flink.sql.parser:*</include>
<include>org.codehaus.janino:*</include>
<include>joda-time:*</include>
+ <include>joda-convert:*</include>
</includes>
</artifactSet>
<relocations>
@@ -371,8 +372,8 @@ under the License.
<!-- flink-table-planner dependencies -->
<relocation>
- <pattern>org.joda.time</pattern>
- <shadedPattern>org.apache.flink.table.shaded.org.joda.time</shadedPattern>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.table.shaded.org.joda</shadedPattern>
</relocation>
<!-- not relocated for now, because we need to change the contents of the properties field otherwise -->
<!--<relocation>
[flink] 01/03: [hotfix][table-sql-parser] Removed unnecessary
dependency on guava.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7c202b07b8b7b884e7d2d2d3501c36344f574e79
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 1 11:06:07 2019 +0200
[hotfix][table-sql-parser] Removed unnecessary dependency on guava.
---
flink-table/flink-sql-parser/pom.xml | 5 -----
.../src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java | 5 ++---
2 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/flink-table/flink-sql-parser/pom.xml b/flink-table/flink-sql-parser/pom.xml
index d0f7ecd..2c5a7ce 100644
--- a/flink-table/flink-sql-parser/pom.xml
+++ b/flink-table/flink-sql-parser/pom.xml
@@ -40,11 +40,6 @@ under the License.
<dependencies>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-guava</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<!-- When updating the Calcite version, make sure to update the dependency exclusions -->
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
index 566de41..b77af04 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
@@ -21,8 +21,6 @@ package org.apache.flink.sql.parser.ddl;
import org.apache.flink.sql.parser.ExtendedSqlNode;
import org.apache.flink.sql.parser.error.SqlParseException;
-import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
-
import org.apache.calcite.sql.SqlCharStringLiteral;
import org.apache.calcite.sql.SqlCreate;
import org.apache.calcite.sql.SqlIdentifier;
@@ -34,6 +32,7 @@ import org.apache.calcite.sql.SqlSpecialOperator;
import org.apache.calcite.sql.SqlWriter;
import org.apache.calcite.sql.parser.SqlParserPos;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -63,7 +62,7 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode {
@Override
public List<SqlNode> getOperandList() {
- List<SqlNode> ops = Lists.newArrayList();
+ List<SqlNode> ops = new ArrayList<>();
ops.add(viewName);
ops.add(fieldList);
ops.add(query);