You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2022/12/08 14:08:51 UTC
[incubator-kyuubi] branch branch-1.6 updated: [KYUUBI #3928] Refer the YarnApplicationState and FinalApplicationStatus to map application state
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/branch-1.6 by this push:
new 283f1e1e6 [KYUUBI #3928] Refer the YarnApplicationState and FinalApplicationStatus to map application state
283f1e1e6 is described below
commit 283f1e1e68a34d7f26cf7df1a7052cd922c57daa
Author: fwang12 <fw...@ebay.com>
AuthorDate: Thu Dec 8 22:08:33 2022 +0800
[KYUUBI #3928] Refer the YarnApplicationState and FinalApplicationStatus to map application state
### _Why are the changes needed?_
We shall respect the final application status first.
Otherwise, the batch state is not consisted with the app state.
FYI:
<img width="498" alt="image" src="https://user-images.githubusercontent.com/6757692/206176219-838eebcc-8a0b-479f-847b-46e56b568aa7.png">

This pr refer the livy implementation:
https://github.com/apache/incubator-livy/blob/50faa9f68a6f1c79f605a4cb70e452f9c7f21528/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala#L236-L259
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3928 from turboFei/yarn_final.
Closes #3928
0e251d9f0 [fwang12] refer livy
dd3aeef9c [fwang12] wait FinalApplicationStatus
0c14a213f [fwang12] save
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
(cherry picked from commit 27b525b5d6f69ccc70b72f388315583587ca6697)
Signed-off-by: fwang12 <fw...@ebay.com>
---
.../kyuubi/engine/YarnApplicationOperation.scala | 43 ++++++++++++++--------
1 file changed, 28 insertions(+), 15 deletions(-)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
index 2c8c165bc..b38b1daa2 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
@@ -19,7 +19,7 @@ package org.apache.kyuubi.engine
import scala.collection.JavaConverters._
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.api.records.{FinalApplicationStatus, YarnApplicationState}
import org.apache.hadoop.yarn.client.api.YarnClient
import org.apache.kyuubi.Logging
@@ -87,7 +87,10 @@ class YarnApplicationOperation extends ApplicationOperation with Logging {
val info = ApplicationInfo(
id = report.getApplicationId.toString,
name = report.getName,
- state = toApplicationState(report.getYarnApplicationState),
+ state = toApplicationState(
+ report.getApplicationId.toString,
+ report.getYarnApplicationState,
+ report.getFinalApplicationStatus),
url = Option(report.getTrackingUrl),
error = Option(report.getDiagnostics))
debug(s"Successfully got application info by $tag: $info")
@@ -110,18 +113,28 @@ class YarnApplicationOperation extends ApplicationOperation with Logging {
}
object YarnApplicationOperation extends Logging {
- def toApplicationState(state: YarnApplicationState): ApplicationState = state match {
- case YarnApplicationState.NEW => ApplicationState.PENDING
- case YarnApplicationState.NEW_SAVING => ApplicationState.PENDING
- case YarnApplicationState.SUBMITTED => ApplicationState.PENDING
- case YarnApplicationState.ACCEPTED => ApplicationState.PENDING
- case YarnApplicationState.RUNNING => ApplicationState.RUNNING
- case YarnApplicationState.FINISHED => ApplicationState.FINISHED
- case YarnApplicationState.FAILED => ApplicationState.FAILED
- case YarnApplicationState.KILLED => ApplicationState.KILLED
- case _ =>
- warn(s"The yarn driver state: $state is not supported, " +
- "mark the application state as UNKNOWN.")
- ApplicationState.UNKNOWN
+ def toApplicationState(
+ appId: String,
+ yarnAppState: YarnApplicationState,
+ finalAppStatus: FinalApplicationStatus): ApplicationState = {
+ (yarnAppState, finalAppStatus) match {
+ case (YarnApplicationState.NEW, FinalApplicationStatus.UNDEFINED) |
+ (YarnApplicationState.NEW_SAVING, FinalApplicationStatus.UNDEFINED) |
+ (YarnApplicationState.SUBMITTED, FinalApplicationStatus.UNDEFINED) |
+ (YarnApplicationState.ACCEPTED, FinalApplicationStatus.UNDEFINED) =>
+ ApplicationState.PENDING
+ case (YarnApplicationState.RUNNING, FinalApplicationStatus.UNDEFINED) |
+ (YarnApplicationState.RUNNING, FinalApplicationStatus.SUCCEEDED) =>
+ ApplicationState.RUNNING
+ case (YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED) =>
+ ApplicationState.FINISHED
+ case (YarnApplicationState.FAILED, FinalApplicationStatus.FAILED) =>
+ ApplicationState.FAILED
+ case (YarnApplicationState.KILLED, FinalApplicationStatus.KILLED) =>
+ ApplicationState.KILLED
+ case (state, finalStatus) => // any other combination is invalid, so FAIL the application.
+ error(s"Unknown YARN state $state for app $appId with final status $finalStatus.")
+ ApplicationState.FAILED
+ }
}
}