You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/03/21 22:38:21 UTC
[7/8] flink git commit: [FLINK-9020][E2ETests] Use separate modules
per testcase
[FLINK-9020][E2ETests] Use separate modules per testcase
This closes #5717.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/129e2159
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/129e2159
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/129e2159
Branch: refs/heads/master
Commit: 129e215961895ef3c759b4fbb919a3be2ce2f1a0
Parents: 91707e3
Author: Florian Schmidt <fl...@icloud.com>
Authored: Mon Mar 19 13:26:14 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Mar 21 21:01:52 2018 +0100
----------------------------------------------------------------------
.../pom.xml | 108 +++++++++++++++++++
.../flink/runtime/taskmanager/TaskManager.java | 30 ++++++
.../streaming/tests/ClassLoaderTestProgram.java | 102 ++++++++++++++++++
.../src/main/resources/.version.properties | 1 +
flink-end-to-end-tests/pom.xml | 78 ++------------
.../flink/runtime/taskmanager/TaskManager.java | 30 ------
.../streaming/tests/ClassLoaderTestProgram.java | 102 ------------------
.../src/main/resources/.version.properties | 1 -
.../test-scripts/test_streaming_classloader.sh | 4 +-
9 files changed, 249 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
new file mode 100644
index 0000000..ee43515
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <artifactId>flink-end-to-end-tests</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>1.6-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>flink-parent-child-classloading-test_${scala.binary.version}</artifactId>
+ <name>flink-parent-child-classloading-test</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+
+ <executions>
+ <!-- ClassLoaderTestProgram -->
+ <execution>
+ <id>ClassLoaderTestProgram</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>ClassLoaderTestProgram</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.tests.ClassLoaderTestProgram</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/tests/ClassLoaderTestProgram.class</include>
+ <include>org/apache/flink/runtime/taskmanager/TaskManager.class</include>
+ <include>.version.properties</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--simplify the name of the testing JARs for referring to them in the end-to-end test scripts-->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>rename</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy file="${project.basedir}/target/flink-parent-child-classloading-test_${scala.binary.version}-${project.version}-ClassLoaderTestProgram.jar" tofile="${project.basedir}/target/ClassLoaderTestProgram.jar" />
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
new file mode 100644
index 0000000..3626885
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+/**
+ * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this
+ * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do
+ * so.
+ */
+public class TaskManager {
+ public static String getMessage() {
+ return "Hello, World!";
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
new file mode 100644
index 0000000..1d4ca4c
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.tests;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Enumeration;
+import java.util.Properties;
+
+/**
+ * End-to-end test program for verifying that the {@code classloader.resolve-order} setting
+ * is being honored by Flink. We test this by creating a fake {@code TaskManager} with a single
+ * method that we call in the same package as the original Flink {@code TaskManager} and verify that
+ * we get a {@link NoSuchMethodError} if we're running with {@code parent-first} class loading
+ * and that we get the correct result from the method when we're running with {@code child-first}
+ * class loading.
+ */
+public class ClassLoaderTestProgram {
+
+ public static void main(String[] args) throws Exception {
+
+ final ParameterTool params = ParameterTool.fromArgs(args);
+
+ final String resolveOrder = params.getRequired("resolve-order");
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ env
+ .fromElements("Hello")
+ .map((MapFunction<String, String>) value -> {
+
+ String gitUrl;
+
+ try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream(".version.properties")) {
+ Properties properties = new Properties();
+ properties.load(propFile);
+ gitUrl = properties.getProperty("git.remote.origin.url");
+ }
+
+ Enumeration<URL> resources = ClassLoaderTestProgram.class.getClassLoader().getResources(
+ ".version.properties");
+
+ StringBuilder sortedProperties = new StringBuilder();
+ while (resources.hasMoreElements()) {
+ URL url = resources.nextElement();
+ try (InputStream in = url.openStream()) {
+ Properties properties = new Properties();
+ properties.load(in);
+ String orderedGitUrl = properties.getProperty("git.remote.origin.url");
+ sortedProperties.append(orderedGitUrl);
+ }
+ }
+
+ if (resolveOrder.equals("parent-first")) {
+ try {
+ @SuppressWarnings("unused")
+ String ignored = TaskManager.getMessage();
+
+ throw new RuntimeException(
+ "TaskManager.getMessage() should not be available with parent-first " +
+ "ClassLoader order.");
+
+ } catch (NoSuchMethodError e) {
+ // expected
+ }
+ return "NoSuchMethodError:" + gitUrl + ":" + sortedProperties;
+ } else if (resolveOrder.equals("child-first")) {
+ String message = TaskManager.getMessage();
+ if (!message.equals("Hello, World!")) {
+ throw new RuntimeException("Wrong message from fake TaskManager.");
+ }
+ return message + ":" + gitUrl + ":" + sortedProperties;
+ } else {
+ throw new RuntimeException("Unknown resolve order: " + resolveOrder);
+ }
+ })
+ .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
+
+ env.execute("ClassLoader Test Program");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
new file mode 100644
index 0000000..dc98aea
--- /dev/null
+++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
@@ -0,0 +1 @@
+git.remote.origin.url=hello-there-42
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
index a5bbc52..32a6e78 100644
--- a/flink-end-to-end-tests/pom.xml
+++ b/flink-end-to-end-tests/pom.xml
@@ -29,79 +29,13 @@ under the License.
<relativePath>..</relativePath>
</parent>
- <artifactId>flink-end-to-end-tests_${scala.binary.version}</artifactId>
- <name>flink-end-to-end-tests</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.4</version>
+ <packaging>pom</packaging>
- <executions>
- <!-- ClassLoaderTestProgram -->
- <execution>
- <id>ClassLoaderTestProgram</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>ClassLoaderTestProgram</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.tests.ClassLoaderTestProgram</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/tests/ClassLoaderTestProgram.class</include>
- <include>org/apache/flink/runtime/taskmanager/TaskManager.class</include>
- <include>.version.properties</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
+ <artifactId>flink-end-to-end-tests</artifactId>
+ <name>flink-end-to-end-tests</name>
- <!--simplify the name of the testing JARs for referring to them in the end-to-end test scripts-->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <id>rename</id>
- <phase>package</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <copy file="${project.basedir}/target/flink-end-to-end-tests_${scala.binary.version}-${project.version}-ClassLoaderTestProgram.jar" tofile="${project.basedir}/target/ClassLoaderTestProgram.jar" />
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <modules>
+ <module>flink-parent-child-classloading-test</module>
+ </modules>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
deleted file mode 100644
index 3626885..0000000
--- a/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-/**
- * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this
- * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do
- * so.
- */
-public class TaskManager {
- public static String getMessage() {
- return "Hello, World!";
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
deleted file mode 100644
index 1d4ca4c..0000000
--- a/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.tests;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.runtime.taskmanager.TaskManager;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Enumeration;
-import java.util.Properties;
-
-/**
- * End-to-end test program for verifying that the {@code classloader.resolve-order} setting
- * is being honored by Flink. We test this by creating a fake {@code TaskManager} with a single
- * method that we call in the same package as the original Flink {@code TaskManager} and verify that
- * we get a {@link NoSuchMethodError} if we're running with {@code parent-first} class loading
- * and that we get the correct result from the method when we're running with {@code child-first}
- * class loading.
- */
-public class ClassLoaderTestProgram {
-
- public static void main(String[] args) throws Exception {
-
- final ParameterTool params = ParameterTool.fromArgs(args);
-
- final String resolveOrder = params.getRequired("resolve-order");
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- env
- .fromElements("Hello")
- .map((MapFunction<String, String>) value -> {
-
- String gitUrl;
-
- try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream(".version.properties")) {
- Properties properties = new Properties();
- properties.load(propFile);
- gitUrl = properties.getProperty("git.remote.origin.url");
- }
-
- Enumeration<URL> resources = ClassLoaderTestProgram.class.getClassLoader().getResources(
- ".version.properties");
-
- StringBuilder sortedProperties = new StringBuilder();
- while (resources.hasMoreElements()) {
- URL url = resources.nextElement();
- try (InputStream in = url.openStream()) {
- Properties properties = new Properties();
- properties.load(in);
- String orderedGitUrl = properties.getProperty("git.remote.origin.url");
- sortedProperties.append(orderedGitUrl);
- }
- }
-
- if (resolveOrder.equals("parent-first")) {
- try {
- @SuppressWarnings("unused")
- String ignored = TaskManager.getMessage();
-
- throw new RuntimeException(
- "TaskManager.getMessage() should not be available with parent-first " +
- "ClassLoader order.");
-
- } catch (NoSuchMethodError e) {
- // expected
- }
- return "NoSuchMethodError:" + gitUrl + ":" + sortedProperties;
- } else if (resolveOrder.equals("child-first")) {
- String message = TaskManager.getMessage();
- if (!message.equals("Hello, World!")) {
- throw new RuntimeException("Wrong message from fake TaskManager.");
- }
- return message + ":" + gitUrl + ":" + sortedProperties;
- } else {
- throw new RuntimeException("Unknown resolve order: " + resolveOrder);
- }
- })
- .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
-
- env.execute("ClassLoader Test Program");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/resources/.version.properties
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/src/main/resources/.version.properties b/flink-end-to-end-tests/src/main/resources/.version.properties
deleted file mode 100644
index dc98aea..0000000
--- a/flink-end-to-end-tests/src/main/resources/.version.properties
+++ /dev/null
@@ -1 +0,0 @@
-git.remote.origin.url=hello-there-42
http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
index 95c58f8..34c55f7 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh
@@ -19,7 +19,7 @@
source "$(dirname "$0")"/common.sh
-TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/target/ClassLoaderTestProgram.jar
+TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-parent-child-classloading-test/target/ClassLoaderTestProgram.jar
echo "Testing parent-first class loading"
@@ -111,4 +111,4 @@ if [[ "$OUTPUT" != "$EXPECTED" ]]; then
echo -e "EXPECTED: $EXPECTED"
echo -e "ACTUAL: $OUTPUT"
PASS=""
-fi
\ No newline at end of file
+fi