You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2019/08/02 13:19:37 UTC
[flink] branch master updated (5254126 -> 3fa4898)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 5254126 [FLINK-13544][connectors] Set parallelism of table sink operator to input transformation parallelism
new ca61ebc [hotfix][table-sql-parser] Removed unnecessary dependency on guava.
new 0673018 [hotfix][table-planner] Added joda.convert to planner jar
new 3fa4898 [FLINK-13504][table] Fixed shading issues in table modules.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../test-scripts/test_table_shaded_dependencies.sh | 111 +++++++++++++++++++++
flink-table/flink-sql-parser/pom.xml | 5 -
.../apache/flink/sql/parser/ddl/SqlCreateView.java | 5 +-
flink-table/flink-table-planner-blink/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 7 +-
flink-table/flink-table-uber-blink/pom.xml | 2 +-
flink-table/flink-table-uber/pom.xml | 3 +-
8 files changed, 122 insertions(+), 15 deletions(-)
create mode 100755 flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
[flink] 02/03: [hotfix][table-planner] Added joda.convert to
planner jar
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 06730181bc70eb160202aff4d88b14abbd01e100
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Fri Aug 2 09:56:48 2019 +0200
[hotfix][table-planner] Added joda.convert to planner jar
---
flink-table/flink-table-planner/pom.xml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index c1f8a1f..d4c00d0 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -342,6 +342,7 @@ under the License.
<include>org.apache.flink.sql.parser:*</include>
<include>org.codehaus.janino:*</include>
<include>joda-time:*</include>
+ <include>joda-convert:*</include>
</includes>
</artifactSet>
<relocations>
@@ -371,8 +372,8 @@ under the License.
<!-- flink-table-planner dependencies -->
<relocation>
- <pattern>org.joda.time</pattern>
- <shadedPattern>org.apache.flink.table.shaded.org.joda.time</shadedPattern>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.table.shaded.org.joda</shadedPattern>
</relocation>
<!-- not relocated for now, because we need to change the contents of the properties field otherwise -->
<!--<relocation>
[flink] 01/03: [hotfix][table-sql-parser] Removed unnecessary
dependency on guava.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit ca61ebcb1f680510a55481e757f0637f4a2c39cd
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 1 11:06:07 2019 +0200
[hotfix][table-sql-parser] Removed unnecessary dependency on guava.
---
flink-table/flink-sql-parser/pom.xml | 5 -----
.../src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java | 5 ++---
2 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/flink-table/flink-sql-parser/pom.xml b/flink-table/flink-sql-parser/pom.xml
index 51bcaf1..a4c9b5d 100644
--- a/flink-table/flink-sql-parser/pom.xml
+++ b/flink-table/flink-sql-parser/pom.xml
@@ -40,11 +40,6 @@ under the License.
<dependencies>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-guava</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<!-- When updating the Calcite version, make sure to update the dependency exclusions -->
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
index 566de41..b77af04 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java
@@ -21,8 +21,6 @@ package org.apache.flink.sql.parser.ddl;
import org.apache.flink.sql.parser.ExtendedSqlNode;
import org.apache.flink.sql.parser.error.SqlParseException;
-import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
-
import org.apache.calcite.sql.SqlCharStringLiteral;
import org.apache.calcite.sql.SqlCreate;
import org.apache.calcite.sql.SqlIdentifier;
@@ -34,6 +32,7 @@ import org.apache.calcite.sql.SqlSpecialOperator;
import org.apache.calcite.sql.SqlWriter;
import org.apache.calcite.sql.parser.SqlParserPos;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -63,7 +62,7 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode {
@Override
public List<SqlNode> getOperandList() {
- List<SqlNode> ops = Lists.newArrayList();
+ List<SqlNode> ops = new ArrayList<>();
ops.add(viewName);
ops.add(fieldList);
ops.add(query);
[flink] 03/03: [FLINK-13504][table] Fixed shading issues in table
modules.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3fa4898a4d1b2d69ef64f48b5a5aefafa2e03974
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 1 11:00:14 2019 +0200
[FLINK-13504][table] Fixed shading issues in table modules.
Properly include the flink-sql-parser into flink-planner* modules shaded
jars. Removed the flink-sql-parser module from flink-table-uber* jars as
it is already included in the flink-planner* modules.
This closes #9313.
---
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../test-scripts/test_table_shaded_dependencies.sh | 111 +++++++++++++++++++++
flink-table/flink-table-planner-blink/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 2 +-
flink-table/flink-table-uber-blink/pom.xml | 2 +-
flink-table/flink-table-uber/pom.xml | 3 +-
6 files changed, 117 insertions(+), 5 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index d50b328..a97b6a0 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -158,6 +158,8 @@ run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_h
run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25"
+run_test "Dependency shading of table modules test" "$END_TO_END_DIR/test-scripts/test_table_shaded_dependencies.sh"
+
################################################################################
# Sticky Scheduling
################################################################################
diff --git a/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
new file mode 100755
index 0000000..cdbec5f
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_table_shaded_dependencies.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+FLINK_SCALA_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target" | sed -n "s/.*flink-table-runtime-blink_\(.*\)-tests\.jar/\1/p" | uniq`
+FLINK_VERSION=`ls "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target" | sed -n "s/.*flink-table-api-java-\(.*\)-tests\.jar/\1/p" | uniq`
+
+# This checks the bytecode for dependencies on external classes. Some classes below
+# are not available in the uber jar. We check for classes that we expect in the uber jar with
+# checkAllowedPackages method.
+function checkCodeDependencies {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jdeps $JAR |\
+ grep "^\s*\->" |\
+ `# jdk dependencies` \
+ grep -v "^\s*\-> java." |\
+ grep -v "^\s*\-> sun.misc." |\
+ grep -v "^\s*\-> javax." |\
+ `# scala dependencies` \
+ grep -v "^\s*\-> scala" |\
+ `# flink dependencies` \
+ grep -v "^\s*\-> org.apache.flink" |\
+ `# janino dependencies` \
+ grep -v "^\s*\-> org.codehaus.janino" |\
+ grep -v "^\s*\-> org.codehaus.commons" |\
+ grep -v "^\s*\-> org.apache.tools.ant" |\
+ `# calcite dependencies` \
+ grep -v "^\s*\-> org.apache.calcite" |\
+ grep -v "^\s*\-> org.pentaho.aggdes" |\
+ grep -v "^\s*\-> org.apache.commons.lang3" |\
+ grep -v "^\s*\-> org.apache.commons.math3" |\
+ grep -v "^\s*\-> org.apache.commons.dbcp2" |\
+ grep -v "^\s*\-> org.apache.http" |\
+ grep -v "^\s*\-> org.w3c.dom" |\
+ grep -v "^\s*\-> org.xml.sax" |\
+ grep -v "^\s*\-> org.ietf.jgss" |\
+ grep -v "^\s*\-> com.esri.core." |\
+ grep -v "^\s*\-> com.yahoo.sketches.hll." |\
+ grep -v "^\s*\-> org.slf4j" |\
+ grep -v "^\s*\-> org.json" |\
+ grep -v "^\s*\-> org.apache.tapestry5.json." |\
+ grep -v "^\s*\-> org.codehaus.jettison" |\
+ grep -v "^\s*\-> net.minidev.json" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted dependencies in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted dependencies in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+# Checks that the uber jars contain only flink, relocated packages, or packages that we
+# consciously decided to include as not relocated.
+function checkAllowedPackages {
+ local JAR=$1
+ local CONTENTS_FILE=$TEST_DATA_DIR/contentsInJar.txt
+
+ jar tf $JAR |\
+ grep ".*class" |\
+ grep -v "org/codehaus/janino" |\
+ grep -v "org/codehaus/commons" |\
+ grep -v "org/apache/calcite" |\
+ grep -v "org/apache/flink" > $CONTENTS_FILE
+ if [[ `cat $CONTENTS_FILE | wc -l` -eq '0' ]]; then
+ echo "Success: There are no unwanted classes in the ${JAR} jar."
+ else
+ echo "Failure: There are unwanted classes in the ${JAR} jar: `cat $CONTENTS_FILE`"
+ exit 1
+ fi
+}
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkCodeDependencies "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java/target/flink-table-api-java-${FLINK_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala/target/flink-table-api-scala_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-java-bridge/target/flink-table-api-java-bridge_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-api-scala-bridge/target/flink-table-api-scala-bridge_${FLINK_SCALA_VERSION}.jar"
+
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner/target/flink-table-planner_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-planner-blink/target/flink-table-planner-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${END_TO_END_DIR}/../flink-table/flink-table-runtime-blink/target/flink-table-runtime-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table-blink_${FLINK_SCALA_VERSION}.jar"
+checkAllowedPackages "${FLINK_DIR}/lib/flink-table_${FLINK_SCALA_VERSION}.jar"
diff --git a/flink-table/flink-table-planner-blink/pom.xml b/flink-table/flink-table-planner-blink/pom.xml
index bd6ce79..f9e7f2b 100644
--- a/flink-table/flink-table-planner-blink/pom.xml
+++ b/flink-table/flink-table-planner-blink/pom.xml
@@ -336,7 +336,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner-blink dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<!-- flink-table-runtime-blink dependencies -->
<include>org.codehaus.janino:*</include>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index d4c00d0..9ab8f19 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -339,7 +339,7 @@ under the License.
<include>commons-codec:commons-codec</include>
<!-- flink-table-planner dependencies -->
- <include>org.apache.flink.sql.parser:*</include>
+ <include>org.apache.flink:flink-sql-parser</include>
<include>org.codehaus.janino:*</include>
<include>joda-time:*</include>
<include>joda-convert:*</include>
diff --git a/flink-table/flink-table-uber-blink/pom.xml b/flink-table/flink-table-uber-blink/pom.xml
index ffd40a4..32517ba 100644
--- a/flink-table/flink-table-uber-blink/pom.xml
+++ b/flink-table/flink-table-uber-blink/pom.xml
@@ -102,8 +102,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>
diff --git a/flink-table/flink-table-uber/pom.xml b/flink-table/flink-table-uber/pom.xml
index aa2c434..394417d 100644
--- a/flink-table/flink-table-uber/pom.xml
+++ b/flink-table/flink-table-uber/pom.xml
@@ -86,7 +86,6 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
- <!-- Exclude all flink-dist files and only include flink-table-* -->
<execution>
<id>shade-flink</id>
<phase>package</phase>
@@ -96,8 +95,8 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
+ <!--Sql parser is included in planners-->
<include>org.apache.flink:flink-table-common</include>
- <include>org.apache.flink:flink-sql-parser</include>
<include>org.apache.flink:flink-table-api-java</include>
<include>org.apache.flink:flink-table-api-scala_${scala.binary.version}</include>
<include>org.apache.flink:flink-table-api-java-bridge_${scala.binary.version}</include>