You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2019/01/18 11:20:13 UTC
[flink] branch master updated: [FLINK-11036] Fix
test_streaming_classloader.sh end-to-end test
This is an automated email from the ASF dual-hosted git repository.
aljoscha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new de04658 [FLINK-11036] Fix test_streaming_classloader.sh end-to-end test
de04658 is described below
commit de04658819d1a38d561a19802c8972a8c241e1c2
Author: Aljoscha Krettek <al...@gmail.com>
AuthorDate: Tue Jan 15 10:12:55 2019 +0100
[FLINK-11036] Fix test_streaming_classloader.sh end-to-end test
Before, this was relying on .version.properties being available in the
flink-dist jar. This was not the case when building from a Flink source
release.
This is fixed by introducing a fake lib package that we place in the
lib/ folder to test the classloader resolution order. Both the lib
package and the user package contain a parent-child-test.properties and
we check the resolution order in the test.
Also change the test to not use TaskManager anymore.
This test was relying on TaskManager being present in Flink, this might
not be the case in the future. We change this test to place a fake class
in the "lib" package and the "user" package to make it independent of
the rest of Flink.
---
.../pom.xml | 27 ++++-------
.../tests/ParentChildTestingVehicle.java} | 12 ++---
.../main/resources/parent-child-test.properties | 19 ++++++++
.../pom.xml | 9 +++-
.../streaming/tests/ClassLoaderTestProgram.java | 55 +++++++---------------
.../tests/ParentChildTestingVehicle.java} | 12 ++---
.../main/resources/parent-child-test.properties | 19 ++++++++
.../src/main/resources/.version.properties | 1 -
flink-end-to-end-tests/pom.xml | 3 +-
.../test-scripts/test_streaming_classloader.sh | 38 +++++++++++----
10 files changed, 113 insertions(+), 82 deletions(-)
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/pom.xml
similarity index 69%
copy from flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
copy to flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/pom.xml
index dbe133c..760c310 100644
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/pom.xml
@@ -21,6 +21,11 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ This is used by the end-to-end test test_streaming_classloader.sh. It contains the library
+ package that we put in the lib/ folder.
+ -->
+
<parent>
<artifactId>flink-end-to-end-tests</artifactId>
<groupId>org.apache.flink</groupId>
@@ -30,19 +35,10 @@
<modelVersion>4.0.0</modelVersion>
- <artifactId>flink-parent-child-classloading-test_${scala.binary.version}</artifactId>
- <name>flink-parent-child-classloading-test</name>
+ <artifactId>flink-parent-child-classloading-test-lib-package_${scala.binary.version}</artifactId>
+ <name>flink-parent-child-classloading-test-lib-package</name>
<packaging>jar</packaging>
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
<build>
<plugins>
<plugin>
@@ -50,18 +46,13 @@
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
- <id>ClassLoaderTestProgram</id>
+ <id>LibPackage</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
- <finalName>ClassLoaderTestProgram</finalName>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.flink.streaming.tests.ClassLoaderTestProgram</mainClass>
- </transformer>
- </transformers>
+ <finalName>LibPackage</finalName>
</configuration>
</execution>
</executions>
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
similarity index 70%
copy from flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
copy to flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
index 3626885..7cad178 100644
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.flink.runtime.taskmanager;
+package org.apache.flink.streaming.tests;
/**
- * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this
- * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do
- * so.
+ * This is used for test_streaming_classloader.sh. We have a version of this in a "lib" package that
+ * we place in the lib/ folder of Flink and a version in the "user" package. We check how they are
+ * resolved in ClassLoaderTestProgram.
*/
-public class TaskManager {
+public class ParentChildTestingVehicle {
public static String getMessage() {
- return "Hello, World!";
+ return "Hello, from lib package!";
}
}
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/resources/parent-child-test.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/resources/parent-child-test.properties
new file mode 100644
index 0000000..f5e2e4a
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-lib-package/src/main/resources/parent-child-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+message=hello-from-lib-package
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/pom.xml
similarity index 88%
rename from flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
rename to flink-end-to-end-tests/flink-parent-child-classloading-test-program/pom.xml
index dbe133c..cf7bcfe 100644
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/pom.xml
@@ -21,6 +21,11 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ This is used by the end-to-end test test_streaming_classloader.sh. It contains the user program
+ that we want to test.
+ -->
+
<parent>
<artifactId>flink-end-to-end-tests</artifactId>
<groupId>org.apache.flink</groupId>
@@ -30,8 +35,8 @@
<modelVersion>4.0.0</modelVersion>
- <artifactId>flink-parent-child-classloading-test_${scala.binary.version}</artifactId>
- <name>flink-parent-child-classloading-test</name>
+ <artifactId>flink-parent-child-classloading-test-program_${scala.binary.version}</artifactId>
+ <name>flink-parent-child-classloading-test-program</name>
<packaging>jar</packaging>
<dependencies>
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
similarity index 55%
rename from flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
rename to flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
index 15ea735..83f766b 100644
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.tests;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.runtime.taskmanager.TaskManager;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import java.io.InputStream;
@@ -29,12 +28,11 @@ import java.util.Enumeration;
import java.util.Properties;
/**
- * End-to-end test program for verifying that the {@code classloader.resolve-order} setting
- * is being honored by Flink. We test this by creating a fake {@code TaskManager} with a single
- * method that we call in the same package as the original Flink {@code TaskManager} and verify that
- * we get a {@link NoSuchMethodError} if we're running with {@code parent-first} class loading
- * and that we get the correct result from the method when we're running with {@code child-first}
- * class loading.
+ * End-to-end test program for verifying that the {@code classloader.resolve-order} setting is being
+ * honored by Flink. We test this by creating a {@code ParentChildTestingVehicle} with a single
+ * method that we call in the same package as the {@code ParentChildTestingVehicle} in the "lib"
+ * package (flink-parent-child-classloading-test-lib-package) and verify the message in the
+ * end-to-end test script.
*/
public class ClassLoaderTestProgram {
@@ -42,58 +40,39 @@ public class ClassLoaderTestProgram {
final ParameterTool params = ParameterTool.fromArgs(args);
- final String resolveOrder = params.getRequired("resolve-order");
-
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env
.fromElements("Hello")
.map((MapFunction<String, String>) value -> {
- String gitCommitId;
+ String messageFromPropsFile;
- try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream(".version.properties")) {
+ try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream("parent-child-test.properties")) {
Properties properties = new Properties();
properties.load(propFile);
- gitCommitId = properties.getProperty("git.commit.id.abbrev");
+ messageFromPropsFile = properties.getProperty("message");
}
+ // Enumerate all properties files we can find and store the messages in the
+ // order we find them. The order will be different between parent-first and
+ // child-first classloader mode.
Enumeration<URL> resources = ClassLoaderTestProgram.class.getClassLoader().getResources(
- ".version.properties");
+ "parent-child-test.properties");
- StringBuilder sortedProperties = new StringBuilder();
+ StringBuilder orderedProperties = new StringBuilder();
while (resources.hasMoreElements()) {
URL url = resources.nextElement();
try (InputStream in = url.openStream()) {
Properties properties = new Properties();
properties.load(in);
- String orderedGitCommitId = properties.getProperty("git.commit.id.abbrev");
- sortedProperties.append(orderedGitCommitId);
+ String messageFromEnumeratedPropsFile = properties.getProperty("message");
+ orderedProperties.append(messageFromEnumeratedPropsFile);
}
}
- if (resolveOrder.equals("parent-first")) {
- try {
- @SuppressWarnings("unused")
- String ignored = TaskManager.getMessage();
-
- throw new RuntimeException(
- "TaskManager.getMessage() should not be available with parent-first " +
- "ClassLoader order.");
-
- } catch (NoSuchMethodError e) {
- // expected
- }
- return "NoSuchMethodError:" + gitCommitId + ":" + sortedProperties;
- } else if (resolveOrder.equals("child-first")) {
- String message = TaskManager.getMessage();
- if (!message.equals("Hello, World!")) {
- throw new RuntimeException("Wrong message from fake TaskManager.");
- }
- return message + ":" + gitCommitId + ":" + sortedProperties;
- } else {
- throw new RuntimeException("Unknown resolve order: " + resolveOrder);
- }
+ String message = ParentChildTestingVehicle.getMessage();
+ return message + ":" + messageFromPropsFile + ":" + orderedProperties;
})
.writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
similarity index 70%
rename from flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
rename to flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
index 3626885..997983d 100644
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/java/org/apache/flink/streaming/tests/ParentChildTestingVehicle.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.flink.runtime.taskmanager;
+package org.apache.flink.streaming.tests;
/**
- * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this
- * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do
- * so.
+ * This is used for test_streaming_classloader.sh. We have a version of this in a "lib" package that
+ * we place in the lib/ folder of Flink and a version in the "user" package. We check how they are
+ * resolved in ClassLoaderTestProgram.
*/
-public class TaskManager {
+public class ParentChildTestingVehicle {
public static String getMessage() {
- return "Hello, World!";
+ return "Hello, from user package!";
}
}
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/resources/parent-child-test.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/resources/parent-child-test.properties
new file mode 100644
index 0000000..ce65ca5
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test-program/src/main/resources/parent-child-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+message=hello-from-user-package
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
deleted file mode 100644
index 76f9c5a..0000000
--- a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
+++ /dev/null
@@ -1 +0,0 @@
-git.commit.id.abbrev=hello-there-42
diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
index a942e74..f6c36dc 100644
--- a/flink-end-to-end-tests/pom.xml
+++ b/flink-end-to-end-tests/pom.xml
@@ -36,7 +36,8 @@ under the License.
<modules>
<module>flink-cli-test</module>
- <module>flink-parent-child-classloading-test</module>
+ <module>flink-parent-child-classloading-test-program</module>
+ <module>flink-parent-child-classloading-test-lib-package</module>
<module>flink-dataset-allround-test</module>
<module>flink-datastream-allround-test</module>
<module>flink-stream-sql-test</module>
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
index 358b80c..66404f1 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
@@ -19,7 +19,7 @@
source "$(dirname "$0")"/common.sh
-TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-parent-child-classloading-test/target/ClassLoaderTestProgram.jar
+TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-parent-child-classloading-test-program/target/ClassLoaderTestProgram.jar
echo "Testing parent-first class loading"
@@ -28,6 +28,24 @@ sed -i -e 's/classloader.resolve-order: .*//' "$FLINK_DIR/conf/flink-conf.yaml"
sed -i -e 's/classloader.parent-first-patterns: .*//' $FLINK_DIR/conf/flink-conf.yaml
echo "classloader.resolve-order: parent-first" >> "$FLINK_DIR/conf/flink-conf.yaml"
+echo "Moving fake LibPackage.jar from end-to-end tests to lib/"
+cp ${END_TO_END_DIR}/flink-parent-child-classloading-test-lib-package/target/LibPackage.jar ${FLINK_DIR}/lib/
+
+function classloader_cleanup() {
+ # don't call ourselves again for another signal interruption
+ trap "exit -1" INT
+ # don't call ourselves again for normal exit
+ trap "" EXIT
+
+ stop_cluster
+ $FLINK_DIR/bin/taskmanager.sh stop-all
+
+ # remove LibPackage.jar again
+ rm ${FLINK_DIR}/lib/LibPackage.jar
+}
+trap classloader_cleanup INT
+trap classloader_cleanup EXIT
+
start_cluster
$FLINK_DIR/bin/flink run -p 1 $TEST_PROGRAM_JAR --resolve-order parent-first --output $TEST_DATA_DIR/out/cl_out_pf
@@ -39,9 +57,9 @@ sed -i -e 's/classloader.resolve-order: .*//' $FLINK_DIR/conf/flink-conf.yaml
OUTPUT=`cat $TEST_DATA_DIR/out/cl_out_pf`
# first field: whether we found the method on TaskManager
-# result of getResource(".version.properties"), should be from the parent
-# ordered result of getResources(".version.properties"), should have parent first
-EXPECTED="NoSuchMethodError:[0-9a-f]{6,}:[0-9a-f]{6,}hello-there-42"
+# result of getResource("parent-child-test.properties"), should be from the parent
+# ordered result of getResources("parent-child-test.properties"), should have parent first
+EXPECTED="Hello, from lib package!:hello-from-lib-package:hello-from-lib-packagehello-from-user-package"
if ! [[ "$OUTPUT" =~ $EXPECTED ]]; then
echo "Output from Flink program does not match expected output."
echo -e "EXPECTED: $EXPECTED"
@@ -69,9 +87,9 @@ sed -i -e 's/classloader.resolve-order: .*//' $FLINK_DIR/conf/flink-conf.yaml
OUTPUT=`cat $TEST_DATA_DIR/out/cl_out_cf_pf`
# first field: whether we found the method on TaskManager
-# result of getResource(".version.properties"), should be from the child
-# ordered result of getResources(".version.properties"), should be child first
-EXPECTED="NoSuchMethodError:hello-there-42:hello-there-42[0-9a-f]{6,}"
+# result of getResource("parent-child-test.properties"), should be from the child
+# ordered result of getResources("parent-child-test.properties"), should be child first
+EXPECTED="Hello, from lib package!:hello-from-user-package:hello-from-user-packagehello-from-lib-package"
if ! [[ "$OUTPUT" =~ $EXPECTED ]]; then
echo "Output from Flink program does not match expected output."
echo -e "EXPECTED: $EXPECTED"
@@ -98,9 +116,9 @@ sed -i -e 's/classloader.parent-first-patterns: .*//' $FLINK_DIR/conf/flink-conf
OUTPUT=`cat $TEST_DATA_DIR/out/cl_out_cf`
# first field: whether we found the method on TaskManager
-# result of getResource(".version.properties"), should be from the child
-# ordered result of getResources(".version.properties"), should be child first
-EXPECTED="Hello, World!:hello-there-42:hello-there-42[0-9a-f]{6,}"
+# result of getResource("parent-child-test.properties"), should be from the child
+# ordered result of getResources("parent-child-test.properties"), should be child first
+EXPECTED="Hello, from user package!:hello-from-user-package:hello-from-user-packagehello-from-lib-package"
if ! [[ "$OUTPUT" =~ $EXPECTED ]]; then
echo "Output from Flink program does not match expected output."
echo -e "EXPECTED: $EXPECTED"