You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2022/07/20 01:14:17 UTC
[brooklyn-server] branch master updated: misc improvements to container tasks
This is an automated email from the ASF dual-hosted git repository.
heneveld pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
The following commit(s) were added to refs/heads/master by this push:
new 3bc62fdc29 misc improvements to container tasks
3bc62fdc29 is described below
commit 3bc62fdc290f0d3404028229ab8541b13867258d
Author: Alex Heneveld <al...@cloudsoft.io>
AuthorDate: Wed Jul 20 01:59:35 2022 +0100
misc improvements to container tasks
* fix namespace pattern (mustn't end in number, nor be too long; now uses Shortener)
* move CTResult to top level
* make default pull policy IfNotPresent
* better reporting for common errors, eg image not found, and test
---
.../brooklyn/tasks/kubectl/ContainerCommons.java | 2 +-
.../brooklyn/tasks/kubectl/ContainerEffector.java | 2 +-
.../brooklyn/tasks/kubectl/ContainerSensor.java | 2 +-
.../tasks/kubectl/ContainerTaskFactory.java | 36 ++++++++----------
.../tasks/kubectl/ContainerTaskResult.java | 44 ++++++++++++++++++++++
.../brooklyn/tasks/kubectl/ContainerTaskTest.java | 42 +++++++++++++++------
6 files changed, 93 insertions(+), 35 deletions(-)
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
index 5dbdf41af5..c35cb7ec0f 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
@@ -33,7 +33,7 @@ import java.util.Set;
public interface ContainerCommons {
ConfigKey<String> CONTAINER_IMAGE = ConfigKeys.newStringConfigKey("image", "Container image");
ConfigKey<PullPolicy> CONTAINER_IMAGE_PULL_POLICY = ConfigKeys.newConfigKey(new TypeToken<PullPolicy>() {} ,
- "imagePullPolicy", "Container image pull policy. Allowed values: {IfNotPresent, Always, Never}. ", PullPolicy.ALWAYS);
+ "imagePullPolicy", "Container image pull policy. Allowed values: {IfNotPresent, Always, Never}. Default IfNotPresent.", PullPolicy.IF_NOT_PRESENT);
ConfigKey<Boolean> KEEP_CONTAINER_FOR_DEBUGGING = ConfigKeys.newBooleanConfigKey("keepContainerForDebugging", "When set to true, the namespace" +
" and associated resources and services are not destroyed after execution. Defaults value is 'false'.", Boolean.FALSE);
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
index efc9950028..e551d6166e 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
@@ -66,7 +66,7 @@ public class ContainerEffector extends AddEffectorInitializerAbstract implements
@Override
public String call(ConfigBag parameters) {
ConfigBag configBag = ConfigBag.newInstanceCopying(this.params).putAll(parameters);
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Executing Container Image: " + EntityInitializers.resolve(configBag, CONTAINER_IMAGE))
.jobIdentifier(entity().getId() + "-" + EFFECTOR_TAG)
.configure(configBag.getAllConfig())
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
index 66c9e94eef..626429cb4a 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
@@ -75,7 +75,7 @@ public class ContainerSensor<T> extends AbstractAddSensorFeed<T> implements Cont
.callable(new Callable<Object>() {
@Override
public Object call() throws Exception {
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running " + EntityInitializers.resolve(configBag, SENSOR_NAME))
.jobIdentifier(entity.getId() + "-" + SENSOR_TAG)
.configure(configBag.getAllConfig())
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
index 45b17058fa..74c95c48fd 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
@@ -40,6 +40,8 @@ import org.apache.brooklyn.util.core.task.system.ProcessTaskWrapper;
import org.apache.brooklyn.util.core.task.system.SimpleProcessTaskFactory;
import org.apache.brooklyn.util.core.task.system.internal.SystemProcessTaskFactory;
import org.apache.brooklyn.util.exceptions.Exceptions;
+import org.apache.brooklyn.util.text.Identifiers;
+import org.apache.brooklyn.util.text.StringShortener;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.CountdownTimer;
import org.apache.brooklyn.util.time.Duration;
@@ -60,7 +62,7 @@ import java.util.stream.Collectors;
import static org.apache.brooklyn.tasks.kubectl.ContainerCommons.*;
-public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> implements SimpleProcessTaskFactory<T,ContainerTaskFactory.ContainerTaskResult,RET, Task<RET>> {
+public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> implements SimpleProcessTaskFactory<T, ContainerTaskResult,RET, Task<RET>> {
private static final Logger LOG = LoggerFactory.getLogger(ContainerTaskFactory.class);
@@ -111,9 +113,12 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
final String cleanImageName = containerImage.contains(":") ? containerImage.substring(0, containerImage.indexOf(":")) : containerImage;
- final String containerName = (Strings.isNonBlank(this.jobIdentifier) ? this.jobIdentifier + "-" : "")
- .concat(cleanImageName).concat("-").concat(Strings.makeRandomId(10))
- .replaceAll("[^A-Za-z0-9-]", "") // remove all symbols
+ StringShortener ss = new StringShortener().separator("-");
+ if (Strings.isNonBlank(this.jobIdentifier)) ss.append("job", this.jobIdentifier).canTruncate("job", 10);
+ ss.append("image", cleanImageName).canTruncate("image", 10);
+ ss.append("uid", Strings.makeRandomId(9)+Identifiers.makeRandomPassword(1, Identifiers.LOWER_CASE_ALPHA));
+ final String containerName = ss.getStringOfMaxLength(50)
+ .replaceAll("[^A-Za-z0-9-]+", "-") // remove all symbols
.toLowerCase();
if (namespace==null) {
namespace = "brooklyn-" + containerName;
@@ -235,10 +240,15 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
throw new IllegalStateException("Timeout waiting for success or failure");
// any other one-off checks for job error, we could do here
+ // e.g. if image can't be pulled for instance
// finally get the partial log for reporting
- ProcessTaskWrapper<String> outputSoFarCmd = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, containerName, namespace)).summary("Retrieve output so far").newTask());
+ ProcessTaskWrapper<String> outputSoFarCmd = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, containerName, namespace)).summary("Retrieve output so far").allowingNonZeroExitCode().newTask());
BrooklynTaskTags.setTransient(outputSoFarCmd.asTask());
+ outputSoFarCmd.block();
+ if (outputSoFarCmd.getExitCode()!=0) {
+ throw new IllegalStateException("Error detected with container job while reading logs (exit code "+outputSoFarCmd.getExitCode()+"): "+outputSoFarCmd.getStdout() + " / "+outputSoFarCmd.getStderr());
+ }
String outputSoFar = outputSoFarCmd.get();
String newOutput = outputSoFar.substring(stdout.size());
LOG.debug("Container job "+namespace+" output: "+newOutput);
@@ -383,20 +393,4 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
}
}
- public static class ContainerTaskResult {
- private List<ProcessTaskWrapper<?>> interestingJobs;
- private String mainStdout;
- private Integer mainExitCode;
-
- /** This will be 0 unless allowNonZeroExitCode was specified */
- public Integer getMainExitCode() {
- return mainExitCode;
- }
- public String getMainStdout() {
- return mainStdout;
- }
- public List<ProcessTaskWrapper<?>> getInterestingJobs() {
- return interestingJobs;
- }
- }
}
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java
new file mode 100644
index 0000000000..3eba0b5898
--- /dev/null
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.tasks.kubectl;
+
+import org.apache.brooklyn.util.core.task.system.ProcessTaskWrapper;
+
+import java.util.List;
+
+public class ContainerTaskResult {
+ List<ProcessTaskWrapper<?>> interestingJobs;
+ String mainStdout;
+ Integer mainExitCode;
+
+ /**
+ * This will be 0 unless allowNonZeroExitCode was specified
+ */
+ public Integer getMainExitCode() {
+ return mainExitCode;
+ }
+
+ public String getMainStdout() {
+ return mainStdout;
+ }
+
+ public List<ProcessTaskWrapper<?>> getInterestingJobs() {
+ return interestingJobs;
+ }
+}
diff --git a/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java b/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
index 828f9be068..04ec550933 100644
--- a/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
+++ b/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
@@ -47,7 +47,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
public void testSuccessfulContainerTask() {
TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running container task")
.jobIdentifier("test-container-task")
.imagePullPolicy(PullPolicy.IF_NOT_PRESENT)
@@ -57,7 +57,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
- ContainerTaskFactory.ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
+ ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
Asserts.assertEquals(result.getMainStdout().trim(), "hello test");
}
@@ -65,7 +65,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
public void testContainerTaskWithVar() {
TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running container task")
.jobIdentifier("test-container-task")
.imagePullPolicy(PullPolicy.IF_NOT_PRESENT)
@@ -76,7 +76,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
- ContainerTaskFactory.ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
+ ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
Asserts.assertEquals(result.getMainStdout().trim(), "hello EnvTest");
Asserts.assertEquals(BrooklynTaskTags.stream(containerTask, BrooklynTaskTags.STREAM_ENV).streamContents.get().trim(), "test_name=\"EnvTest\"");
}
@@ -103,7 +103,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
public void testExpectedFailingContainerTask() {
TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running container task")
.jobIdentifier("test-container-task")
.imagePullPolicy(PullPolicy.IF_NOT_PRESENT)
@@ -125,7 +125,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
public void testSleepingAndExpectedFailingContainerTask() {
TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running container task")
.jobIdentifier("test-container-task")
.imagePullPolicy(PullPolicy.IF_NOT_PRESENT)
@@ -159,7 +159,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
public void testNotExpectedFailingContainerTask() {
TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running container task")
.jobIdentifier("test-container-task")
.imagePullPolicy(PullPolicy.IF_NOT_PRESENT)
@@ -170,7 +170,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
- Asserts.assertTrue(containerTask.blockUntilEnded(Duration.ONE_MINUTE)); // should complete in 1 minute, i.e. we detect the failed
+ Asserts.assertTrue(containerTask.blockUntilEnded(Duration.THIRTY_SECONDS)); // should complete quickly when we detect the failed
Asserts.assertTrue(containerTask.isDone());
Asserts.assertTrue(containerTask.isError());
Asserts.assertFailsWith(() -> containerTask.getUnchecked(), error -> Asserts.expectedFailureContainsIgnoreCase(error, "Non-zero", "42"));
@@ -202,7 +202,7 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
try {
// first create the script
- Task<ContainerTaskFactory.ContainerTaskResult> setup = baseFactory.bashScriptCommands(
+ Task<ContainerTaskResult> setup = baseFactory.bashScriptCommands(
"mkdir -p /brooklyn-mount-dir/scripts",
"cd /brooklyn-mount-dir/scripts",
"echo echo hello " + uid + " > hello-"+uid+".sh",
@@ -211,12 +211,12 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
DynamicTasks.queueIfPossible(setup).orSubmitAsync(entity).getTask().getUnchecked();
// now make a new container that should see the same mount point, and try running the command
- Task<ContainerTaskFactory.ContainerTaskResult> containerTask = baseFactory.bashScriptCommands(
+ Task<ContainerTaskResult> containerTask = baseFactory.bashScriptCommands(
"./hello-"+uid+".sh"
)
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
- ContainerTaskFactory.ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
+ ContainerTaskResult result = containerTask.getUnchecked(Duration.ONE_MINUTE);
Asserts.assertEquals(result.getMainStdout().trim(), "hello " + uid);
} finally {
@@ -224,4 +224,24 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
}
}
+ @Test
+ public void testImageNotAvailable() {
+ TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
+
+ Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
+ .summary("Invalid image test")
+ .jobIdentifier("test-container-task")
+ .imagePullPolicy(PullPolicy.NEVER)
+ .timeout(Duration.TWO_MINUTES)
+ .image("image-should-not-exist-"+Identifiers.makeRandomId(4))
+ .command( "bad-command-too" )
+ .allowingNonZeroExitCode()
+ .newTask();
+ DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
+
+ Asserts.assertTrue(containerTask.blockUntilEnded(Duration.THIRTY_SECONDS)); // should complete quickly when we detect the failed
+ Asserts.assertTrue(containerTask.isDone());
+ Asserts.assertTrue(containerTask.isError());
+ Asserts.assertFailsWith(() -> containerTask.getUnchecked(), error -> Asserts.expectedFailureContainsIgnoreCase(error, "image"));
+ }
}