You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2020/05/12 13:57:32 UTC
[flink-docker] branch dev-1.10 updated (5ca8c22 -> 9e96a9e)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch dev-1.10
in repository https://gitbox.apache.org/repos/asf/flink-docker.git.
from 5ca8c22 [FLINK-17439] Fork dev-1.10 branch
new 0b0d6f1 [hotfix] Remove diff generation
new 9e96a9e [FLINK-17164] Add job-cluster support
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 4 +-
.travis.yml | 7 +-
docker-entrypoint.sh | 21 ++++--
testing/docker-test-job/pom.xml | 75 ++++++++++++++++++++++
.../main/java/org/apache/flink/StreamingJob.java | 48 ++++++++++++++
testing/run_travis_tests.sh | 8 ---
testing/testing_lib.sh | 41 ++++++++++--
7 files changed, 183 insertions(+), 21 deletions(-)
create mode 100644 testing/docker-test-job/pom.xml
create mode 100644 testing/docker-test-job/src/main/java/org/apache/flink/StreamingJob.java
[flink-docker] 02/02: [FLINK-17164] Add job-cluster support
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch dev-1.10
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
commit 9e96a9e78a8f6979458202ffb497ee53a5a93815
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Apr 22 12:23:33 2020 +0200
[FLINK-17164] Add job-cluster support
---
.gitignore | 4 +-
.travis.yml | 7 +-
docker-entrypoint.sh | 21 ++++--
testing/docker-test-job/pom.xml | 75 ++++++++++++++++++++++
.../main/java/org/apache/flink/StreamingJob.java | 48 ++++++++++++++
testing/testing_lib.sh | 41 ++++++++++--
6 files changed, 183 insertions(+), 13 deletions(-)
diff --git a/.gitignore b/.gitignore
index feb151f..825fb4d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
.*.swp
.idea
-dev
\ No newline at end of file
+dev
+*.iml
+testing/docker-test-job/target
diff --git a/.travis.yml b/.travis.yml
index a4e4f1d..6f58f78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,11 @@
os: linux
dist: bionic
-language: shell
+language: java
+jdk: "openjdk8"
+
+cache:
+ directories:
+ - $HOME/.m2
services:
- docker
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index 011ccea..6084f06 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -18,6 +18,8 @@
# limitations under the License.
###############################################################################
+COMMAND_STANDALONE="standalone-job"
+
# If unspecified, the hostname of the container is taken as the JobManager address
JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml"
@@ -77,11 +79,7 @@ set_common_options() {
set_config_option query.server.port 6125
}
-if [ "$1" = "help" ]; then
- echo "Usage: $(basename "$0") (jobmanager|taskmanager|help)"
- exit 0
-elif [ "$1" = "jobmanager" ]; then
- shift 1
+prepare_job_manager_start() {
echo "Starting Job Manager"
copy_plugins_if_required
@@ -91,8 +89,21 @@ elif [ "$1" = "jobmanager" ]; then
echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
fi
envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+}
+
+if [ "$1" = "help" ]; then
+ echo "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|help)"
+ exit 0
+elif [ "$1" = "jobmanager" ]; then
+ shift 1
+ prepare_job_manager_start
exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
+elif [ "$1" = ${COMMAND_STANDALONE} ]; then
+ shift 1
+ prepare_job_manager_start
+
+ exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "$@"
elif [ "$1" = "taskmanager" ]; then
shift 1
echo "Starting Task Manager"
diff --git a/testing/docker-test-job/pom.xml b/testing/docker-test-job/pom.xml
new file mode 100644
index 0000000..6c6185f
--- /dev/null
+++ b/testing/docker-test-job/pom.xml
@@ -0,0 +1,75 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.flink</groupId>
+ <artifactId>docker-test-job</artifactId>
+ <version>1.0</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <flink.version>1.9.0</flink.version>
+ <java.version>1.8</java.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <!-- Java Compiler -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/testing/docker-test-job/src/main/java/org/apache/flink/StreamingJob.java b/testing/docker-test-job/src/main/java/org/apache/flink/StreamingJob.java
new file mode 100644
index 0000000..4806a43
--- /dev/null
+++ b/testing/docker-test-job/src/main/java/org/apache/flink/StreamingJob.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink;
+
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+
+public class StreamingJob {
+
+ public static void main(String[] args) throws Exception {
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.addSource(new InfiniteSource()).map(x -> x);
+ env.execute();
+ }
+
+ private static final class InfiniteSource implements SourceFunction<Integer> {
+
+ private volatile boolean running = true;
+
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ while (running) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ running = false;
+ }
+ }
+}
diff --git a/testing/testing_lib.sh b/testing/testing_lib.sh
index 92f524d..f619791 100644
--- a/testing/testing_lib.sh
+++ b/testing/testing_lib.sh
@@ -60,7 +60,7 @@ function internal_run() {
}
function internal_run_jobmanager() {
- internal_run "$1" "--name jobmanager --publish 6123:6123 --publish 8081:8081 $2" jobmanager
+ internal_run "$1" "--name jobmanager --publish 6123:6123 --publish 8081:8081 $2" "$3"
}
function internal_run_taskmanager() {
@@ -144,6 +144,10 @@ function test_image() {
echo >&2 "===> ${image_tag} taskmanager connected."
}
+function build_test_job() {
+ mvn package -f testing/docker-test-job/pom.xml
+}
+
function create_network() {
docker network create "$NETWORK_NAME" > /dev/null
}
@@ -172,23 +176,48 @@ function cleanup() {
fi
}
+function internal_smoke_test() {
+ local dockerfile=$1
+ local jm_docker_run_command_args=$2
+ local jm_command_args=$3
+ local tm_docker_run_command_args=$4
+
+ jobmanager_container_id="$(internal_run_jobmanager \
+ "$dockerfile" \
+ "${jm_docker_run_command_args}" \
+ "${jm_command_args}")"
+ taskmanager_container_id="$(internal_run_taskmanager \
+ "$dockerfile" \
+ "${tm_docker_run_command_args}")"
+ wait_for_jobmanager "$dockerfile"
+ test_image "$dockerfile"
+ docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null
+}
+
function internal_smoke_test_images() {
local dockerfiles="$1"
local docker_run_command_args="$2"
create_network
trap cleanup EXIT RETURN
+ build_test_job
local jobmanager_container_id
local taskmanager_container_id
for dockerfile in $dockerfiles; do
build_image "$dockerfile"
- jobmanager_container_id="$(internal_run_jobmanager "$dockerfile" "${docker_run_command_args}")"
- taskmanager_container_id="$(internal_run_taskmanager "$dockerfile" "${docker_run_command_args}")"
- wait_for_jobmanager "$dockerfile"
- test_image "$dockerfile"
- docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null
+
+ internal_smoke_test \
+ "$dockerfile" \
+ "${docker_run_command_args}" \
+ "jobmanager" \
+ "${docker_run_command_args}"
+ internal_smoke_test \
+ "$dockerfile" \
+ "${docker_run_command_args} --mount type=bind,src=$(pwd)/testing/docker-test-job/target,target=/opt/flink/usrlib" \
+ "standalone-job --job-classname org.apache.flink.StreamingJob" \
+ "${docker_run_command_args} --mount type=bind,src=$(pwd)/testing/docker-test-job/target,target=/opt/flink/usrlib"
done
}
[flink-docker] 01/02: [hotfix] Remove diff generation
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch dev-1.10
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
commit 0b0d6f142939224cb65707356d48810c2b6a8378
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Apr 30 12:05:01 2020 +0200
[hotfix] Remove diff generation
---
testing/run_travis_tests.sh | 8 --------
1 file changed, 8 deletions(-)
diff --git a/testing/run_travis_tests.sh b/testing/run_travis_tests.sh
index 8ace2b7..508bc62 100755
--- a/testing/run_travis_tests.sh
+++ b/testing/run_travis_tests.sh
@@ -11,14 +11,6 @@ fi
BRANCH="$TRAVIS_BRANCH"
-if [ -n "$IS_PULL_REQUEST" ]; then
- changed_files="$(git diff --name-only $BRANCH...HEAD)"
-
- echo "Changed files in this pull request:"
- echo "${changed_files}"
- echo
-fi
-
./add-version.sh -r 1.10 -f 1.10.0
if [ -z "$IS_PULL_REQUEST" ] && [ "$BRANCH" = "dev-1.10" ]; then