You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2022/07/20 12:20:47 UTC
[brooklyn-server] branch master updated: better logging for DST errors and container errors
This is an automated email from the ASF dual-hosted git repository.
heneveld pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
The following commit(s) were added to refs/heads/master by this push:
new aed1038c43 better logging for DST errors and container errors
aed1038c43 is described below
commit aed1038c4333ee7d29feccdfab0cbc5d6909ceed
Author: Alex Heneveld <al...@cloudsoft.io>
AuthorDate: Wed Jul 20 13:20:31 2022 +0100
better logging for DST errors and container errors
---
.../util/core/task/DynamicSequentialTask.java | 12 +-
.../brooklyn/tasks/kubectl/ContainerCommons.java | 11 +-
.../brooklyn/tasks/kubectl/ContainerEffector.java | 2 +-
.../brooklyn/tasks/kubectl/ContainerSensor.java | 2 +-
.../tasks/kubectl/ContainerTaskFactory.java | 130 ++++++++++++++++-----
.../tasks/kubectl/ContainerTaskResult.java | 20 ++--
.../brooklyn/tasks/kubectl/ContainerTaskTest.java | 2 +-
7 files changed, 136 insertions(+), 43 deletions(-)
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/task/DynamicSequentialTask.java b/core/src/main/java/org/apache/brooklyn/util/core/task/DynamicSequentialTask.java
index 5953909d05..d50fd4499d 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/task/DynamicSequentialTask.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/task/DynamicSequentialTask.java
@@ -498,9 +498,15 @@ public class DynamicSequentialTask<T> extends BasicTask<T> implements HasTaskChi
if (throwFirstError) {
if (isError())
getUnchecked();
- for (Task<?> t: getQueue())
- if (t.isError() && !TaskTags.isInessential(t))
- t.getUnchecked();
+ for (Task<?> t: getQueue()) {
+ if (t.isError() && !TaskTags.isInessential(t)) {
+ try {
+ t.getUnchecked();
+ } catch (Exception e) {
+ throw Exceptions.propagate("Error while draining tasks (preceding task in queue failed)", e);
+ }
+ }
+ }
}
}
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
index c35cb7ec0f..d426724e38 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerCommons.java
@@ -59,7 +59,16 @@ public interface ContainerCommons {
String JOBS_FEED_CMD = "kubectl wait --timeout=%ds --for=condition=complete job/%s --namespace=%s";
String JOBS_FEED_FAILED_CMD = "kubectl wait --timeout=%ds --for=condition=failed job/%s --namespace=%s";
String JOBS_LOGS_CMD = "kubectl logs jobs/%s --namespace=%s";
- String PODS_EXIT_CODE_CMD = "kubectl get pods --namespace=%s -ojsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}'";
+ String PODS_CMD_PREFIX = "kubectl get pods --namespace=%s --selector=job-name=%s ";
+ String PODS_STATUS_PHASE_CMD = PODS_CMD_PREFIX + "-ojsonpath='{.items[0].status.phase}'";
+ String PODS_NAME_CMD = PODS_CMD_PREFIX + "-ojsonpath='{.items[0].metadata.name}'";
+ String PODS_EXIT_CODE_CMD = PODS_CMD_PREFIX + "-ojsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}'";
+ String SCOPED_EVENTS_CMD = "kubectl --namespace %s get events --field-selector=involvedObject.name=%s";
+ String SCOPED_EVENTS_FAILED_JSON_CMD = "kubectl --namespace %s get events --field-selector=reason=Failed,involvedObject.name=%s -ojsonpath='{.items}'";
String NAMESPACE_DELETE_CMD = "kubectl delete namespace %s";
+ public static enum PodPhases {
+ // from https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
+ Failed, Running, Succeeded, Unknown, Pending
+ }
}
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
index e551d6166e..d030aee64e 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerEffector.java
@@ -68,7 +68,7 @@ public class ContainerEffector extends AddEffectorInitializerAbstract implements
ConfigBag configBag = ConfigBag.newInstanceCopying(this.params).putAll(parameters);
Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Executing Container Image: " + EntityInitializers.resolve(configBag, CONTAINER_IMAGE))
- .jobIdentifier(entity().getId() + "-" + EFFECTOR_TAG)
+ .jobIdentifier(entity().getApplicationId()+"-"+entity().getId() + "-"+EFFECTOR_TAG)
.configure(configBag.getAllConfig())
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity());
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
index 626429cb4a..d35cac56d9 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerSensor.java
@@ -77,7 +77,7 @@ public class ContainerSensor<T> extends AbstractAddSensorFeed<T> implements Cont
public Object call() throws Exception {
Task<ContainerTaskResult> containerTask = ContainerTaskFactory.newInstance()
.summary("Running " + EntityInitializers.resolve(configBag, SENSOR_NAME))
- .jobIdentifier(entity.getId() + "-" + SENSOR_TAG)
+ .jobIdentifier(entity.getApplication()+"-"+entity.getId() + "-" + SENSOR_TAG)
.configure(configBag.getAllConfig())
.newTask();
DynamicTasks.queueIfPossible(containerTask).orSubmitAsync(entity);
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
index 74c95c48fd..d9736f61a4 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.brooklyn.tasks.kubectl;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import org.apache.brooklyn.api.entity.Entity;
import org.apache.brooklyn.api.mgmt.Task;
@@ -33,6 +34,7 @@ import org.apache.brooklyn.util.core.config.ConfigBag;
import org.apache.brooklyn.util.core.json.ShellEnvironmentSerializer;
import org.apache.brooklyn.util.core.task.DynamicTasks;
import org.apache.brooklyn.util.core.task.TaskBuilder;
+import org.apache.brooklyn.util.core.task.TaskTags;
import org.apache.brooklyn.util.core.task.Tasks;
import org.apache.brooklyn.util.core.task.system.ProcessTaskFactory;
import org.apache.brooklyn.util.core.task.system.ProcessTaskStub;
@@ -57,6 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -80,6 +83,7 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
TaskBuilder<RET> taskBuilder = Tasks.<RET>builder().dynamic(true)
.displayName(this.summary)
.tag(BrooklynTaskTags.tagForStream(BrooklynTaskTags.STREAM_STDOUT, stdout))
+ .tag(new ContainerTaskResult())
.body(()-> {
List<String> commandsCfg = EntityInitializers.resolve(config, COMMAND);
List<String> argumentsCfg = EntityInitializers.resolve(config, ARGUMENTS);
@@ -114,23 +118,29 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
final String cleanImageName = containerImage.contains(":") ? containerImage.substring(0, containerImage.indexOf(":")) : containerImage;
StringShortener ss = new StringShortener().separator("-");
- if (Strings.isNonBlank(this.jobIdentifier)) ss.append("job", this.jobIdentifier).canTruncate("job", 10);
- ss.append("image", cleanImageName).canTruncate("image", 10);
+ if (Strings.isNonBlank(this.jobIdentifier)) {
+ ss.append("job", this.jobIdentifier).canTruncate("job", 20);
+ } else {
+ ss.append("brooklyn", "brooklyn").canTruncate("brooklyn", 2);
+ ss.append("appId", entity.getApplicationId()).canTruncate("appId", 4);
+ ss.append("entityId", entity.getId()).canTruncate("entityId", 4);
+ ss.append("image", cleanImageName).canTruncate("image", 10);
+ }
ss.append("uid", Strings.makeRandomId(9)+Identifiers.makeRandomPassword(1, Identifiers.LOWER_CASE_ALPHA));
- final String containerName = ss.getStringOfMaxLength(50)
+ final String kubeJobName = ss.getStringOfMaxLength(50)
.replaceAll("[^A-Za-z0-9-]+", "-") // remove all symbols
.toLowerCase();
if (namespace==null) {
- namespace = "brooklyn-" + containerName;
+ namespace = kubeJobName;
}
- LOG.debug("Submitting container job in namespace "+namespace);
+ LOG.debug("Submitting container job in namespace "+namespace+", name "+kubeJobName);
Map<String, String> env = new ShellEnvironmentSerializer(((EntityInternal)entity).getManagementContext()).serialize(EntityInitializers.resolve(config, BrooklynConfigKeys.SHELL_ENVIRONMENT));
final BrooklynBomOsgiArchiveInstaller.FileWithTempInfo<File> jobYaml = new KubeJobFileCreator()
.withImage(containerImage)
.withImagePullPolicy(containerImagePullPolicy)
- .withName(containerName)
+ .withName(kubeJobName)
.withCommand(Lists.newArrayList(commandsCfg))
.withArgs(argumentsCfg)
.withEnv(env)
@@ -144,8 +154,18 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
Duration timeout = EntityInitializers.resolve(config, TIMEOUT);
- ContainerTaskResult result = new ContainerTaskResult();
- result.interestingJobs = MutableList.of();
+ ContainerTaskResult result = (ContainerTaskResult) TaskTags.getTagsFast(Tasks.current()).stream().filter(x -> x instanceof ContainerTaskResult).findAny().orElseGet(() -> {
+ LOG.warn("Result object not set on tag at "+Tasks.current()+"; creating");
+ ContainerTaskResult x = new ContainerTaskResult();
+ TaskTags.addTagDynamically(Tasks.current(), x);
+ return x;
+ });
+ result.namespace = namespace;
+ result.kubeJobName = kubeJobName;
+
+ // validate these as they are passed to shell script, prevent injection
+ if (!namespace.matches("[A-Za-z0-9_.-]+")) throw new IllegalStateException("Invalid namespace: "+namespace);
+ if (!kubeJobName.matches("[A-Za-z0-9_.-]+")) throw new IllegalStateException("Invalid job name: "+kubeJobName);
ProcessTaskWrapper<ProcessTaskWrapper<?>> createNsJob = null;
if (!Boolean.FALSE.equals(createNamespace)) {
@@ -181,24 +201,76 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
}
}
- result.interestingJobs.add(DynamicTasks.queue(
- newSimpleTaskFactory(String.format(JOBS_CREATE_CMD, jobYaml.getFile().getAbsolutePath(), namespace)).summary("Submit job").newTask()));
+ DynamicTasks.queue(
+ newSimpleTaskFactory(String.format(JOBS_CREATE_CMD, jobYaml.getFile().getAbsolutePath(), namespace)).summary("Submit job").newTask());
final CountdownTimer timer = CountdownTimer.newInstanceStarted(timeout);
+ // wait for it to be running (or failed / succeeded) -
+ PodPhases podPhase = DynamicTasks.queue(Tasks.<PodPhases>builder().dynamic(true).displayName("Wait for container to be running (or fail)").body(() -> {
+ String phase = null;
+ long first = System.currentTimeMillis();
+ long last = first;
+ long backoffMillis = 10;
+ while (timer.isNotExpired()) {
+ phase = DynamicTasks.queue(newSimpleTaskFactory(String.format(PODS_STATUS_PHASE_CMD, namespace, kubeJobName)).summary("Get pod phase").allowingNonZeroExitCode().newTask()).get().trim();
+ if (PodPhases.Running.name().equalsIgnoreCase(phase)) return PodPhases.Running;
+ if (PodPhases.Failed.name().equalsIgnoreCase(phase)) return PodPhases.Failed;
+ if (PodPhases.Succeeded.name().equalsIgnoreCase(phase)) return PodPhases.Succeeded;
+
+ if (Strings.isNonBlank(phase) && Strings.isBlank(result.kubePodName)) {
+ result.kubePodName = DynamicTasks.queue(newSimpleTaskFactory(String.format(PODS_NAME_CMD, namespace, kubeJobName)).summary("Get pod name").allowingNonZeroExitCode().newTask()).get().trim();
+ }
+ if (PodPhases.Pending.name().equals(phase) && Strings.isNonBlank(result.kubePodName)) {
+ // if pending, look for errors
+ String failedEvents = DynamicTasks.queue(newSimpleTaskFactory(String.format(SCOPED_EVENTS_FAILED_JSON_CMD, namespace, result.kubePodName)).summary("Get pod failed events").allowingNonZeroExitCode().newTask()).get().trim();
+ if (!"[]".equals(failedEvents)) {
+ String events = DynamicTasks.queue(newSimpleTaskFactory(String.format(SCOPED_EVENTS_CMD, namespace, result.kubePodName)).summary("Get pod events").allowingNonZeroExitCode().newTask()).get().trim();
+ throw new IllegalStateException("Job pod failed: "+failedEvents+"\n"+events);
+ }
+ }
+
+ if (System.currentTimeMillis() - last > 10*1000) {
+ last = System.currentTimeMillis();
+ // every 10s log info
+ LOG.info("Container taking long time to start ("+Duration.millis(last-first)+"): "+namespace+" "+kubeJobName+" "+result.kubePodName+" / phase '"+phase+"'");
+ if (Strings.isNonBlank(result.kubePodName)) {
+ String events = DynamicTasks.queue(newSimpleTaskFactory(String.format(SCOPED_EVENTS_CMD, namespace, result.kubePodName)).summary("Get pod events").allowingNonZeroExitCode().newTask()).get().trim();
+ LOG.info("Pod events: \n"+events);
+ } else {
+ String events = DynamicTasks.queue(newSimpleTaskFactory(String.format(SCOPED_EVENTS_CMD, namespace, kubeJobName)).summary("Get job events").allowingNonZeroExitCode().newTask()).get().trim();
+ LOG.info("Job events: \n"+events);
+ }
+ }
+ long backoffMillis2 = backoffMillis;
+ Tasks.withBlockingDetails("waiting "+backoffMillis2+"ms for pod to be available (current status '" + phase + "')", () -> {
+ Time.sleep(backoffMillis2);
+ return null;
+ });
+ if (backoffMillis<80) backoffMillis*=2;
+ }
+ throw new IllegalStateException("Timeout waiting for pod to be available; current status is '" + phase + "'");
+ }).build()).getUnchecked();
+
+ // notify once pod is available
+ synchronized (result) {
+ result.notifyAll();
+ }
+
// use `wait --for` api, but in a 5s loop in case there are other issues
- boolean succeeded = DynamicTasks.queue(Tasks.<Boolean>builder().dynamic(true).displayName("Wait for success or failure").body(() -> {
+ boolean succeeded = podPhase == PodPhases.Succeeded ? true : podPhase == PodPhases.Failed ? false : DynamicTasks.queue(Tasks.<Boolean>builder().dynamic(true).displayName("Wait for success or failure").body(() -> {
while (true) {
LOG.debug("Container job submitted, now waiting on success or failure");
long secondsLeft = Math.min(Math.max(1, timer.getDurationRemaining().toSeconds()), 5);
final AtomicInteger finishCount = new AtomicInteger(0);
- ProcessTaskWrapper<String> waitForSuccess = Entities.submit(entity, newSimpleTaskFactory(String.format(JOBS_FEED_CMD, secondsLeft, containerName, namespace))
+ ProcessTaskWrapper<String> waitForSuccess = Entities.submit(entity, newSimpleTaskFactory(String.format(JOBS_FEED_CMD, secondsLeft, kubeJobName, namespace))
.summary("Wait for success ('complete')").allowingNonZeroExitCode().newTask());
Entities.submit(entity, Tasks.create("Wait for success then notify", () -> {
try {
- if (waitForSuccess.get().contains("condition met")) LOG.debug("Container job "+namespace+" detected as completed (succeeded) in kubernetes");
+ if (waitForSuccess.get().contains("condition met"))
+ LOG.debug("Container job " + namespace + " detected as completed (succeeded) in kubernetes");
} finally {
synchronized (finishCount) {
finishCount.incrementAndGet();
@@ -207,11 +279,12 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
}
}));
- ProcessTaskWrapper<String> waitForFailed = Entities.submit(entity, newSimpleTaskFactory(String.format(JOBS_FEED_FAILED_CMD, secondsLeft, containerName, namespace))
+ ProcessTaskWrapper<String> waitForFailed = Entities.submit(entity, newSimpleTaskFactory(String.format(JOBS_FEED_FAILED_CMD, secondsLeft, kubeJobName, namespace))
.summary("Wait for failed").allowingNonZeroExitCode().newTask());
Entities.submit(entity, Tasks.create("Wait for failed then notify", () -> {
try {
- if (waitForFailed.get().contains("condition met")) LOG.debug("Container job "+namespace+" detected as failed in kubernetes (may be valid non-zero exit)");
+ if (waitForFailed.get().contains("condition met"))
+ LOG.debug("Container job " + namespace + " detected as failed in kubernetes (may be valid non-zero exit)");
} finally {
synchronized (finishCount) {
finishCount.incrementAndGet();
@@ -221,7 +294,7 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
}));
while (finishCount.get() == 0) {
- LOG.debug("Container job "+namespace+" waiting on complete or failed");
+ LOG.debug("Container job " + namespace + " waiting on complete or failed");
try {
synchronized (finishCount) {
finishCount.wait(Duration.TEN_SECONDS.toMilliseconds());
@@ -233,17 +306,13 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
if (waitForSuccess.isDone() && waitForSuccess.getExitCode() == 0) return true;
if (waitForFailed.isDone() && waitForFailed.getExitCode() == 0) return false;
- LOG.debug("Container job "+namespace+" not yet complete, will retry");
- // probably timed out or job not yet available; short wait then retry
- Time.sleep(Duration.millis(50));
- if (timer.isExpired())
- throw new IllegalStateException("Timeout waiting for success or failure");
+ LOG.debug("Container job " + namespace + " not yet complete, will retry");
- // any other one-off checks for job error, we could do here
- // e.g. if image can't be pulled for instance
+ // other one-off checks for job error, we could do here
+ // e.g. if image can't be pulled, for instance
// finally get the partial log for reporting
- ProcessTaskWrapper<String> outputSoFarCmd = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, containerName, namespace)).summary("Retrieve output so far").allowingNonZeroExitCode().newTask());
+ ProcessTaskWrapper<String> outputSoFarCmd = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, kubeJobName, namespace)).summary("Retrieve output so far").allowingNonZeroExitCode().newTask());
BrooklynTaskTags.setTransient(outputSoFarCmd.asTask());
outputSoFarCmd.block();
if (outputSoFarCmd.getExitCode()!=0) {
@@ -253,16 +322,19 @@ public class ContainerTaskFactory<T extends ContainerTaskFactory<T,RET>,RET> imp
String newOutput = outputSoFar.substring(stdout.size());
LOG.debug("Container job "+namespace+" output: "+newOutput);
stdout.write(newOutput.getBytes(StandardCharsets.UTF_8));
+
+ if (timer.isExpired())
+ throw new IllegalStateException("Timeout waiting for success or failure");
+
+ // probably timed out or job not yet available; short wait then retry
+ Time.sleep(Duration.millis(50));
}
}).build()).getUnchecked();
LOG.debug("Container job "+namespace+" completed, success "+succeeded);
- ProcessTaskWrapper<String> retrieveOutput = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, containerName, namespace)).summary("Retrieve output").newTask());
- result.interestingJobs.add(retrieveOutput);
-
- ProcessTaskWrapper<String> retrieveExitCode = DynamicTasks.queue(newSimpleTaskFactory(String.format(PODS_EXIT_CODE_CMD, namespace)).summary("Retrieve exit code").newTask());
- result.interestingJobs.add(retrieveExitCode);
+ ProcessTaskWrapper<String> retrieveOutput = DynamicTasks.queue(newSimpleTaskFactory(String.format(JOBS_LOGS_CMD, kubeJobName, namespace)).summary("Retrieve output").newTask());
+ ProcessTaskWrapper<String> retrieveExitCode = DynamicTasks.queue(newSimpleTaskFactory(String.format(PODS_EXIT_CODE_CMD, namespace, kubeJobName)).summary("Retrieve exit code").newTask());
DynamicTasks.waitForLast();
result.mainStdout = retrieveOutput.get();
diff --git a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java
index 3eba0b5898..22cfe13d46 100644
--- a/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java
+++ b/software/base/src/main/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskResult.java
@@ -18,14 +18,12 @@
*/
package org.apache.brooklyn.tasks.kubectl;
-import org.apache.brooklyn.util.core.task.system.ProcessTaskWrapper;
-
-import java.util.List;
-
public class ContainerTaskResult {
- List<ProcessTaskWrapper<?>> interestingJobs;
String mainStdout;
Integer mainExitCode;
+ String namespace;
+ String kubeJobName;
+ public String kubePodName;
/**
* This will be 0 unless allowNonZeroExitCode was specified
@@ -38,7 +36,15 @@ public class ContainerTaskResult {
return mainStdout;
}
- public List<ProcessTaskWrapper<?>> getInterestingJobs() {
- return interestingJobs;
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public String getKubeJobName() {
+ return kubeJobName;
+ }
+
+ public String getKubePodName() {
+ return kubePodName;
}
}
diff --git a/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java b/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
index 04ec550933..a688ae21b3 100644
--- a/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
+++ b/software/base/src/test/java/org/apache/brooklyn/tasks/kubectl/ContainerTaskTest.java
@@ -242,6 +242,6 @@ public class ContainerTaskTest extends BrooklynAppUnitTestSupport {
Asserts.assertTrue(containerTask.blockUntilEnded(Duration.THIRTY_SECONDS)); // should complete quickly when we detect the failed
Asserts.assertTrue(containerTask.isDone());
Asserts.assertTrue(containerTask.isError());
- Asserts.assertFailsWith(() -> containerTask.getUnchecked(), error -> Asserts.expectedFailureContainsIgnoreCase(error, "image"));
+ Asserts.assertFailsWith(() -> containerTask.getUnchecked(), error -> Asserts.expectedFailureContainsIgnoreCase(error, "job pod failed", "image"));
}
}