You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/01/15 21:32:44 UTC
[hadoop] branch trunk updated: YARN-7200. SLS generates a
realtimetrack.json file but that file is missing the closing ']'.
Contributed by Agshin Kazimli
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6cd540e YARN-7200. SLS generates a realtimetrack.json file but that file is missing the closing ']'. Contributed by Agshin Kazimli
6cd540e is described below
commit 6cd540e96484bb0ce546c12a97cca05da3cb59fc
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Jan 15 22:32:30 2021 +0100
YARN-7200. SLS generates a realtimetrack.json file but that file is missing the closing ']'. Contributed by Agshin Kazimli
---
.../main/java/org/apache/hadoop/yarn/sls/SLSRunner.java | 14 +++++++++-----
.../hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 13 +++++++++++++
.../apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java | 13 +++++++++++++
.../apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java | 4 ++++
4 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index b4d4a80..a8d2aa6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -159,6 +159,10 @@ public class SLSRunner extends Configured implements Tool {
private TraceType inputType;
private SynthTraceJobProducer stjp;
+ public static int getRemainingApps() {
+ return remainingApps;
+ }
+
public SLSRunner() throws ClassNotFoundException {
Configuration tempConf = new Configuration(false);
init(tempConf);
@@ -933,12 +937,12 @@ public class SLSRunner extends Configured implements Tool {
public static void decreaseRemainingApps() {
remainingApps--;
+ }
- if (remainingApps == 0) {
- LOG.info("SLSRunner tears down.");
- if (exitAtTheFinish) {
- System.exit(0);
- }
+ public static void exitSLSRunner() {
+ LOG.info("SLSRunner tears down.");
+ if (exitAtTheFinish) {
+ System.exit(0);
}
}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 9b22390..f34a3d3 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.codahale.metrics.Timer;
@@ -75,6 +77,9 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
private boolean metricsON;
private Tracker tracker;
+ // logger
+ private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
+
public Tracker getTracker() {
return tracker;
}
@@ -218,6 +223,14 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
AppAttemptRemovedSchedulerEvent appRemoveEvent =
(AppAttemptRemovedSchedulerEvent) schedulerEvent;
appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
+ if (SLSRunner.getRemainingApps() == 0) {
+ try {
+ getSchedulerMetrics().tearDown();
+ SLSRunner.exitSLSRunner();
+ } catch (Exception e) {
+ LOG.error("Scheduler Metrics failed to tear down.", e);
+ }
+ }
} else if (schedulerEvent.getType() ==
SchedulerEventType.APP_ATTEMPT_ADDED
&& schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 317c05e..09ef2cf 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashSet;
@@ -63,6 +65,9 @@ public class SLSFairScheduler extends FairScheduler
private Map<ContainerId, Resource> preemptionContainerMap =
new ConcurrentHashMap<>();
+ // logger
+ private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
+
public SchedulerMetrics getSchedulerMetrics() {
return schedulerMetrics;
}
@@ -182,6 +187,14 @@ public class SLSFairScheduler extends FairScheduler
if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
&& schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
SLSRunner.decreaseRemainingApps();
+ if (SLSRunner.getRemainingApps() == 0) {
+ try {
+ getSchedulerMetrics().tearDown();
+ SLSRunner.exitSLSRunner();
+ } catch (Exception e) {
+ LOG.error("Scheduler Metrics failed to tear down.", e);
+ }
+ }
}
}
}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index 2957d23..9f6cf7d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -539,9 +539,13 @@ public abstract class SchedulerMetrics {
}
void tearDown() throws Exception {
+ setRunning(false);
+ LOG.info("Scheduler Metrics tears down");
if (metricsLogBW != null) {
metricsLogBW.write("]");
metricsLogBW.close();
+ //metricsLogBW is nullified to prevent the usage after closing
+ metricsLogBW = null;
}
if (web != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org