You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by qu...@apache.org on 2022/03/26 20:54:19 UTC
[hadoop] branch trunk updated: YARN-11100. Fix StackOverflowError in SLS scheduler event handling. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
quapaw pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new adbaf48 YARN-11100. Fix StackOverflowError in SLS scheduler event handling. Contributed by Szilard Nemeth.
adbaf48 is described below
commit adbaf48082d366bd5f9f0b1baca6b54e76a9d84e
Author: 9uapaw <gy...@gmail.com>
AuthorDate: Sat Mar 26 21:43:10 2022 +0100
YARN-11100. Fix StackOverflowError in SLS scheduler event handling. Contributed by Szilard Nemeth.
---
.../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 5 +++--
.../yarn/sls/scheduler/SLSCapacityScheduler.java | 16 +++++++++++++++-
.../hadoop/yarn/sls/scheduler/SLSFairScheduler.java | 15 +++++++++++++++
.../yarn/sls/scheduler/SLSSchedulerCommons.java | 10 ++++++----
.../hadoop/yarn/sls/scheduler/SchedulerWrapper.java | 19 +++++++++++++++++++
5 files changed, 58 insertions(+), 7 deletions(-)
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 48ad610..2110e3c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -175,11 +175,12 @@ public class SLSRunner extends Configured implements Tool {
}
private void init(Configuration tempConf) throws ClassNotFoundException {
+ // runner configuration
+ setConf(tempConf);
+
nmMap = new ConcurrentHashMap<>();
queueAppNumMap = new HashMap<>();
amRunner = new AMRunner(runner, this);
- // runner configuration
- setConf(tempConf);
// runner
poolSize = tempConf.getInt(SLSConfiguration.RUNNER_POOL_SIZE,
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index b6fe5c0..39170b1 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
@Private
@Unstable
public class SLSCapacityScheduler extends CapacityScheduler implements
- SchedulerWrapper,Configurable {
+ SchedulerWrapper, Configurable {
private final SLSSchedulerCommons schedulerCommons;
private Configuration conf;
@@ -65,6 +65,15 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
}
+ @Override
+ public Allocation allocatePropagated(ApplicationAttemptId attemptId,
+ List<ResourceRequest> resourceRequests,
+ List<SchedulingRequest> schedulingRequests,
+ List<ContainerId> containerIds, List<String> blacklistAdditions,
+ List<String> blacklistRemovals, ContainerUpdates updateRequests) {
+ return super.allocate(attemptId, resourceRequests, schedulingRequests,
+ containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
+ }
@Override
public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
@@ -98,6 +107,11 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
@Override
+ public void propagatedHandle(SchedulerEvent schedulerEvent) {
+ super.handle(schedulerEvent);
+ }
+
+ @Override
public void serviceStop() throws Exception {
schedulerCommons.stopMetrics();
super.serviceStop();
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index b164316..2835b41 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -64,6 +64,21 @@ public class SLSFairScheduler extends FairScheduler
}
@Override
+ public void propagatedHandle(SchedulerEvent schedulerEvent) {
+ super.handle(schedulerEvent);
+ }
+
+ @Override
+ public Allocation allocatePropagated(ApplicationAttemptId attemptId,
+ List<ResourceRequest> resourceRequests,
+ List<SchedulingRequest> schedulingRequests,
+ List<ContainerId> containerIds, List<String> blacklistAdditions,
+ List<String> blacklistRemovals, ContainerUpdates updateRequests) {
+ return super.allocate(attemptId, resourceRequests, schedulingRequests,
+ containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
+ }
+
+ @Override
public void serviceStop() throws Exception {
schedulerCommons.stopMetrics();
super.serviceStop();
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java
index 7132fc9..d83fe5c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java
@@ -100,7 +100,8 @@ public class SLSSchedulerCommons {
.time();
Allocation allocation = null;
try {
- allocation = scheduler.allocate(attemptId, resourceRequests,
+ allocation = ((SchedulerWrapper)scheduler).allocatePropagated(
+ attemptId, resourceRequests,
schedulingRequests, containerIds,
blacklistAdditions, blacklistRemovals, updateRequests);
return allocation;
@@ -118,7 +119,8 @@ public class SLSSchedulerCommons {
}
}
} else {
- return scheduler.allocate(attemptId, resourceRequests, schedulingRequests,
+ return ((SchedulerWrapper)scheduler).allocatePropagated(
+ attemptId, resourceRequests, schedulingRequests,
containerIds,
blacklistAdditions, blacklistRemovals, updateRequests);
}
@@ -204,7 +206,7 @@ public class SLSSchedulerCommons {
public void handle(SchedulerEvent schedulerEvent) {
if (!metricsON) {
- scheduler.handle(schedulerEvent);
+ ((SchedulerWrapper)scheduler).propagatedHandle(schedulerEvent);
return;
}
@@ -245,7 +247,7 @@ public class SLSSchedulerCommons {
operationTimer = schedulerMetrics.getSchedulerHandleTimer(
schedulerEvent.getType()).time();
- scheduler.handle(schedulerEvent);
+ ((SchedulerWrapper)scheduler).propagatedHandle(schedulerEvent);
} finally {
if (handlerTimer != null) {
handlerTimer.stop();
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
index 7112b1a..5ee088d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
@@ -19,7 +19,16 @@ package org.apache.hadoop.yarn.sls.scheduler;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+
+import java.util.List;
@Private
@Unstable
@@ -29,4 +38,14 @@ public interface SchedulerWrapper {
Tracker getTracker();
String getRealQueueName(String queue) throws YarnException;
+
+ void propagatedHandle(SchedulerEvent schedulerEvent);
+
+ Allocation allocatePropagated(ApplicationAttemptId attemptId,
+ List<ResourceRequest> resourceRequests,
+ List<SchedulingRequest> schedulingRequests,
+ List<ContainerId> containerIds,
+ List<String> blacklistAdditions,
+ List<String> blacklistRemovals,
+ ContainerUpdates updateRequests);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org