You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/07/04 05:52:01 UTC
[05/10] flink git commit: [FLINK-9695] Added
mesos.resourcemanager.tasks.container.docker.force-pull-image option
[FLINK-9695] Added mesos.resourcemanager.tasks.container.docker.force-pull-image option
This closes #6232.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b230bf0e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b230bf0e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b230bf0e
Branch: refs/heads/master
Commit: b230bf0e3883ee2dba47e22e781009aa9d0f000e
Parents: 229ed77
Author: Leonid Ishimnikov <li...@fastmail.com>
Authored: Mon Jul 2 11:29:57 2018 -0400
Committer: Till Rohrmann <tr...@apache.org>
Committed: Wed Jul 4 07:51:28 2018 +0200
----------------------------------------------------------------------
.../mesos_task_manager_configuration.html | 5 ++++
docs/ops/deployment/mesos.md | 2 ++
.../clusterframework/LaunchableMesosWorker.java | 3 ++-
.../clusterframework/MesosResourceManager.java | 1 +
.../MesosTaskManagerParameters.java | 21 ++++++++++++++++
.../MesosFlinkResourceManagerTest.java | 1 +
.../MesosResourceManagerTest.java | 2 +-
.../MesosTaskManagerParametersTest.java | 25 ++++++++++++++++++++
8 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/docs/_includes/generated/mesos_task_manager_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/mesos_task_manager_configuration.html b/docs/_includes/generated/mesos_task_manager_configuration.html
index 2c2ed17..0af844d 100644
--- a/docs/_includes/generated/mesos_task_manager_configuration.html
+++ b/docs/_includes/generated/mesos_task_manager_configuration.html
@@ -18,6 +18,11 @@
<td></td>
</tr>
<tr>
+ <td><h5>mesos.resourcemanager.tasks.container.docker.force-pull-image</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Instruct the docker containerizer to forcefully pull the image rather than reuse a cached version.</td>
+ </tr>
+ <tr>
<td><h5>mesos.resourcemanager.tasks.container.docker.parameters</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>Custom parameters to be passed into docker run command when using the docker containerizer. Comma separated list of "key=value" pairs. The "value" may contain '='.</td>
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/docs/ops/deployment/mesos.md
----------------------------------------------------------------------
diff --git a/docs/ops/deployment/mesos.md b/docs/ops/deployment/mesos.md
index 567d9b7..49ee671 100644
--- a/docs/ops/deployment/mesos.md
+++ b/docs/ops/deployment/mesos.md
@@ -270,6 +270,8 @@ May be set to -1 to disable this feature.
`mesos.resourcemanager.tasks.uris`: A comma separated list of URIs of custom artifacts to be downloaded into the sandbox of Mesos workers. (**NO DEFAULT**)
+`mesos.resourcemanager.tasks.container.docker.force-pull-image`: Instruct the docker containerizer to forcefully pull the image rather than reuse a cached version. (**DEFAULT**: false)
+
`mesos.resourcemanager.tasks.hostname`: Optional value to define the TaskManager's hostname. The pattern `_TASK_` is replaced by the actual id of the Mesos task. This can be used to configure the TaskManager to use Mesos DNS (e.g. `_TASK_.flink-service.mesos`) for name lookups. (**NO DEFAULT**)
`mesos.resourcemanager.tasks.bootstrap-cmd`: A command which is executed before the TaskManager is started (**NO DEFAULT**).
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index 0bf09f8..93c90b7 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -317,7 +317,8 @@ public class LaunchableMesosWorker implements LaunchableTask {
.setDocker(Protos.ContainerInfo.DockerInfo.newBuilder()
.addAllParameters(params.dockerParameters())
.setNetwork(Protos.ContainerInfo.DockerInfo.Network.HOST)
- .setImage(params.containerImageName().get()));
+ .setImage(params.containerImageName().get())
+ .setForcePullImage(params.dockerForcePullImage()));
break;
default:
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java
index 4d62c04..ea10ff8 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java
@@ -666,6 +666,7 @@ public class MesosResourceManager extends ResourceManager<RegisteredMesosWorkerN
new HashMap<>(taskManagerParameters.containeredParameters().taskManagerEnv())),
taskManagerParameters.containerVolumes(),
taskManagerParameters.dockerParameters(),
+ taskManagerParameters.dockerForcePullImage(),
taskManagerParameters.constraints(),
taskManagerParameters.command(),
taskManagerParameters.bootstrapCommand(),
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index 1c19cf2..d915b36 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -107,6 +107,12 @@ public class MesosTaskManagerParameters {
.withDescription("Custom parameters to be passed into docker run command when using the docker containerizer." +
" Comma separated list of \"key=value\" pairs. The \"value\" may contain '='.");
+ public static final ConfigOption<Boolean> MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE =
+ key("mesos.resourcemanager.tasks.container.docker.force-pull-image")
+ .defaultValue(false)
+ .withDescription("Instruct the docker containerizer to forcefully pull the image rather than" +
+ " reuse a cached version.");
+
public static final ConfigOption<String> MESOS_CONSTRAINTS_HARD_HOSTATTR =
key("mesos.constraints.hard.hostattribute")
.noDefaultValue()
@@ -135,6 +141,8 @@ public class MesosTaskManagerParameters {
private final List<Protos.Parameter> dockerParameters;
+ private final boolean dockerForcePullImage;
+
private final List<ConstraintEvaluator> constraints;
private final String command;
@@ -153,6 +161,7 @@ public class MesosTaskManagerParameters {
ContaineredTaskManagerParameters containeredParameters,
List<Protos.Volume> containerVolumes,
List<Protos.Parameter> dockerParameters,
+ boolean dockerForcePullImage,
List<ConstraintEvaluator> constraints,
String command,
Option<String> bootstrapCommand,
@@ -166,6 +175,7 @@ public class MesosTaskManagerParameters {
this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
+ this.dockerForcePullImage = dockerForcePullImage;
this.constraints = Preconditions.checkNotNull(constraints);
this.command = Preconditions.checkNotNull(command);
this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
@@ -225,6 +235,13 @@ public class MesosTaskManagerParameters {
}
/**
+ * Get Docker option to force pull image.
+ */
+ public boolean dockerForcePullImage() {
+ return dockerForcePullImage;
+ }
+
+ /**
* Get the placement constraints.
*/
public List<ConstraintEvaluator> constraints() {
@@ -269,6 +286,7 @@ public class MesosTaskManagerParameters {
", containeredParameters=" + containeredParameters +
", containerVolumes=" + containerVolumes +
", dockerParameters=" + dockerParameters +
+ ", dockerForcePullImage=" + dockerForcePullImage +
", constraints=" + constraints +
", taskManagerHostName=" + taskManagerHostname +
", command=" + command +
@@ -329,6 +347,8 @@ public class MesosTaskManagerParameters {
Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));
+ boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);
+
List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);
List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);
@@ -350,6 +370,7 @@ public class MesosTaskManagerParameters {
containeredParameters,
containerVolumes,
dockerParameters,
+ dockerForcePullImage,
constraints,
tmCommand,
tmBootstrapCommand,
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
index 7ee8808..31394b2 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
@@ -251,6 +251,7 @@ public class MesosFlinkResourceManagerTest extends TestLogger {
containeredParams,
Collections.<Protos.Volume>emptyList(),
Collections.<Protos.Parameter>emptyList(),
+ false,
Collections.<ConstraintEvaluator>emptyList(),
"",
Option.<String>empty(),
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java
index 65576bb..9fa8c0e 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java
@@ -279,7 +279,7 @@ public class MesosResourceManagerTest extends TestLogger {
new ContaineredTaskManagerParameters(1024, 768, 256, 4, new HashMap<String, String>());
MesosTaskManagerParameters tmParams = new MesosTaskManagerParameters(
1.0, 1, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams,
- Collections.<Protos.Volume>emptyList(), Collections.<Protos.Parameter>emptyList(),
+ Collections.<Protos.Volume>emptyList(), Collections.<Protos.Parameter>emptyList(), false,
Collections.<ConstraintEvaluator>emptyList(), "", Option.<String>empty(),
Option.<String>empty(), Collections.<String>emptyList());
http://git-wip-us.apache.org/repos/asf/flink/blob/b230bf0e/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
index e002cb9..ead7a12 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
@@ -142,6 +142,31 @@ public class MesosTaskManagerParametersTest extends TestLogger {
assertEquals(params.uris().size(), 0);
}
+ public void testForcePullImageTrue() {
+ Configuration config = new Configuration();
+ config.setBoolean(MesosTaskManagerParameters.MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE, true);
+
+ MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
+ assertEquals(params.dockerForcePullImage(), true);
+ }
+
+ @Test
+ public void testForcePullImageFalse() {
+ Configuration config = new Configuration();
+ config.setBoolean(MesosTaskManagerParameters.MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE, false);
+
+ MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
+ assertEquals(params.dockerForcePullImage(), false);
+ }
+
+ @Test
+ public void testForcePullImageDefault() {
+ Configuration config = new Configuration();
+
+ MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
+ assertEquals(params.dockerForcePullImage(), false);
+ }
+
@Test
public void givenTwoConstraintsInConfigShouldBeParsed() throws Exception {