You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2024/02/08 04:55:18 UTC
(kyuubi) branch master updated: [KYUUBI #6053] [K8S] Show more details for pod/container status in application error
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 0b59fa00c [KYUUBI #6053] [K8S] Show more details for pod/container status in application error
0b59fa00c is described below
commit 0b59fa00c9c693efe2289ffa3ed5bf064764ba4a
Author: Fei Wang <fw...@ebay.com>
AuthorDate: Wed Feb 7 20:55:11 2024 -0800
[KYUUBI #6053] [K8S] Show more details for pod/container status in application error
# :mag: Description
Now the information for k8s app error is too short and confuse.
For example:
```
App Id: spark-8d04e31b6c9540f6952fde81fcc4e19f
App State: FAILED
App Diagnostic: kyuubi-spark-2c7d0d5a-c25b-4b36-bcd0-250c97c90025-driver/spark-kubernetes-driver[Error]
```
We need to provide more information to help debug.
For PodStatus:
```
public String toString() {
return "PodStatus(conditions=" + this.getConditions() + ", containerStatuses=" + this.getContainerStatuses() + ", ephemeralContainerStatuses=" + this.getEphemeralContainerStatuses() + ", hostIP=" + this.getHostIP() + ", initContainerStatuses=" + this.getInitContainerStatuses() + ", message=" + this.getMessage() + ", nominatedNodeName=" + this.getNominatedNodeName() + ", phase=" + this.getPhase() + ", podIP=" + this.getPodIP() + ", podIPs=" + this.getPodIPs() + ", qosClass=" + [...]
}
```
For ContainerState:
```
public String toString() {
return "ContainerState(running=" + this.getRunning() + ", terminated=" + this.getTerminated() + ", waiting=" + this.getWaiting() + ", additionalProperties=" + this.getAdditionalProperties() + ")";
}
```
In this pr, we show the PodStatus or ContainerState directly and provide more useful information.
## Issue References ๐
This pull request fixes #
## Describe Your Solution ๐ง
Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
#### Behavior With This Pull Request :tada:
#### Related Unit Tests
---
# Checklist ๐
- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6053 from turboFei/pod_diagnose.
Closes #6053
34de08c13 [Fei Wang] pod status
f8152e78c [Fei Wang] Show more info
Authored-by: Fei Wang <fw...@ebay.com>
Signed-off-by: Fei Wang <fw...@ebay.com>
---
.../org/apache/kyuubi/client/util/JsonUtils.java | 9 +++++++
.../engine/KubernetesApplicationOperation.scala | 28 ++++++++++++----------
2 files changed, 25 insertions(+), 12 deletions(-)
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/JsonUtils.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/JsonUtils.java
index 855a15280..6d1d58e64 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/JsonUtils.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/JsonUtils.java
@@ -35,6 +35,15 @@ public final class JsonUtils {
}
}
+ public static String toPrettyJson(Object object) {
+ try {
+ return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(object);
+ } catch (Exception e) {
+ throw new KyuubiRestException(
+ String.format("Failed to convert object(%s) to json", object), e);
+ }
+ }
+
public static <T> T fromJson(String json, Class<T> clazz) {
try {
return MAPPER.readValue(json, clazz);
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index 6afe3257b..738e7b23c 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -29,6 +29,7 @@ import io.fabric8.kubernetes.client.KubernetesClient
import io.fabric8.kubernetes.client.informers.{ResourceEventHandler, SharedIndexInformer}
import org.apache.kyuubi.{KyuubiException, Logging, Utils}
+import org.apache.kyuubi.client.util.JsonUtils
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{KubernetesApplicationStateSource, KubernetesCleanupDriverPodStrategy}
import org.apache.kyuubi.config.KyuubiConf.KubernetesApplicationStateSource.KubernetesApplicationStateSource
@@ -364,17 +365,26 @@ object KubernetesApplicationOperation extends Logging {
appStateSource: KubernetesApplicationStateSource,
appStateContainer: String): (ApplicationState, Option[String]) = {
val podName = pod.getMetadata.getName
- val containerStateToBuildAppState = appStateSource match {
+ val containerStatusToBuildAppState = appStateSource match {
case KubernetesApplicationStateSource.CONTAINER =>
pod.getStatus.getContainerStatuses.asScala
- .find(cs => appStateContainer.equalsIgnoreCase(cs.getName)).map(_.getState)
+ .find(cs => appStateContainer.equalsIgnoreCase(cs.getName))
case KubernetesApplicationStateSource.POD => None
}
- val applicationState = containerStateToBuildAppState.map(containerStateToApplicationState)
+ val applicationState = containerStatusToBuildAppState
+ .map(_.getState)
+ .map(containerStateToApplicationState)
.getOrElse(podStateToApplicationState(pod.getStatus.getPhase))
- val applicationError = containerStateToBuildAppState
- .map(cs => containerStateToApplicationError(cs).map(r => s"$podName/$appStateContainer[$r]"))
- .getOrElse(Option(pod.getStatus.getReason).map(r => s"$podName[$r]"))
+ val applicationError = if (ApplicationState.isFailed(applicationState)) {
+ val errorMap = containerStatusToBuildAppState.map { cs =>
+ Map("Pod" -> podName, "Container" -> appStateContainer, "ContainerStatus" -> cs)
+ }.getOrElse {
+ Map("Pod" -> podName, "PodStatus" -> pod.getStatus)
+ }
+ Some(JsonUtils.toPrettyJson(errorMap.asJava))
+ } else {
+ None
+ }
applicationState -> applicationError
}
@@ -393,12 +403,6 @@ object KubernetesApplicationOperation extends Logging {
}
}
- def containerStateToApplicationError(containerState: ContainerState): Option[String] = {
- // https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-states
- Option(containerState.getWaiting).map(_.getReason)
- .orElse(Option(containerState.getTerminated).map(_.getReason))
- }
-
def podStateToApplicationState(podState: String): ApplicationState = podState match {
// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
case "Pending" => PENDING