You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2024/01/30 12:17:26 UTC
(spark) branch master updated: [SPARK-46918][YARN] Replace self-defined variables with Hadoop ContainerExitStatus
This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e32a8249c6dd [SPARK-46918][YARN] Replace self-defined variables with Hadoop ContainerExitStatus
e32a8249c6dd is described below
commit e32a8249c6ddb15e01d2307964f2978f4a10ad56
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Tue Jan 30 20:17:11 2024 +0800
[SPARK-46918][YARN] Replace self-defined variables with Hadoop ContainerExitStatus
### What changes were proposed in this pull request?
Replace the Spark self-defined `VMEM_EXCEEDED_EXIT_CODE` and `PMEM_EXCEEDED_EXIT_CODE` with Hadoop defined `ContainerExitStatus.KILLED_EXCEEDED_VMEM` and `ContainerExitStatus.KILLED_EXCEEDED_PMEM` which were introduced in YARN-2091(since Hadoop 2.5.0)
### Why are the changes needed?
Minor code clean-up
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44950 from pan3793/SPARK-46918.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: yangjie01 <ya...@baidu.com>
---
.../main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 736eaa52b81c..7f0469937fef 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -851,9 +851,6 @@ private[yarn] class YarnAllocator(
onHostStr,
completedContainer.getState,
completedContainer.getExitStatus))
- // Hadoop 2.2.X added a ContainerExitStatus we should switch to use
- // there are some exit status' we shouldn't necessarily count against us, but for
- // now I think its ok as none of the containers are expected to exit.
val exitStatus = completedContainer.getExitStatus
val (exitCausedByApp, containerExitReason) = exitStatus match {
case _ if shutdown =>
@@ -867,7 +864,7 @@ private[yarn] class YarnAllocator(
// just as easily finish on any other executor. See SPARK-8167.
(false, s"Container ${containerId}${onHostStr} was preempted.")
// Should probably still count memory exceeded exit codes towards task failures
- case VMEM_EXCEEDED_EXIT_CODE =>
+ case ContainerExitStatus.KILLED_EXCEEDED_VMEM =>
val vmemExceededPattern = raw"$MEM_REGEX of $MEM_REGEX virtual memory used".r
val diag = vmemExceededPattern.findFirstIn(completedContainer.getDiagnostics)
.map(_.concat(".")).getOrElse("")
@@ -876,7 +873,7 @@ private[yarn] class YarnAllocator(
s"${YarnConfiguration.NM_VMEM_PMEM_RATIO} or disabling " +
s"${YarnConfiguration.NM_VMEM_CHECK_ENABLED} because of YARN-4714."
(true, message)
- case PMEM_EXCEEDED_EXIT_CODE =>
+ case ContainerExitStatus.KILLED_EXCEEDED_PMEM =>
val pmemExceededPattern = raw"$MEM_REGEX of $MEM_REGEX physical memory used".r
val diag = pmemExceededPattern.findFirstIn(completedContainer.getDiagnostics)
.map(_.concat(".")).getOrElse("")
@@ -1025,8 +1022,6 @@ private[yarn] class YarnAllocator(
private object YarnAllocator {
val MEM_REGEX = "[0-9.]+ [KMG]B"
- val VMEM_EXCEEDED_EXIT_CODE = -103
- val PMEM_EXCEEDED_EXIT_CODE = -104
val DECOMMISSIONING_NODES_CACHE_SIZE = 200
val NOT_APP_AND_SYSTEM_FAULT_EXIT_STATUS = Set(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org