You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/04/28 15:52:23 UTC
[hadoop] branch branch-3.2 updated: YARN-10194. YARN RMWebServices
/scheduler-conf/validate leaks ZK Connections. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 1dabbd5 YARN-10194. YARN RMWebServices /scheduler-conf/validate leaks ZK Connections. Contributed by Prabhu Joseph
1dabbd5 is described below
commit 1dabbd500623d663603ba52707a5d94cd153fd0c
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Apr 28 17:52:14 2020 +0200
YARN-10194. YARN RMWebServices /scheduler-conf/validate leaks ZK Connections. Contributed by Prabhu Joseph
---
.../capacity/CapacitySchedulerConfigValidator.java | 18 +++++++++++-------
.../capacity/conf/FSSchedulerConfigurationStore.java | 1 +
.../capacity/conf/InMemoryConfigurationStore.java | 6 ++++++
.../capacity/conf/YarnConfigurationStore.java | 2 +-
.../scheduler/capacity/conf/ZKConfigurationStore.java | 8 ++++++++
5 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index 3957c5f..d8e1285 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -43,14 +43,18 @@ public final class CapacitySchedulerConfigValidator {
public static boolean validateCSConfiguration(
final Configuration oldConf, final Configuration newConf,
final RMContext rmContext) throws IOException {
- //TODO: extract all the validation steps and replace reinitialize with
- //the specific validation steps
CapacityScheduler newCs = new CapacityScheduler();
- newCs.setConf(oldConf);
- newCs.setRMContext(rmContext);
- newCs.init(oldConf);
- newCs.reinitialize(newConf, rmContext, true);
- return true;
+ try {
+ //TODO: extract all the validation steps and replace reinitialize with
+ //the specific validation steps
+ newCs.setConf(oldConf);
+ newCs.setRMContext(rmContext);
+ newCs.init(oldConf);
+ newCs.reinitialize(newConf, rmContext, true);
+ return true;
+ } finally {
+ newCs.stop();
+ }
}
public static Set<String> validatePlacementRules(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 690a60f..c85506f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -348,6 +348,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
return CURRENT_VERSION_INFO;
}
+ @Override
public void close() throws IOException {
if (fileSystem != null) {
fileSystem.close();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 0e11e40..8ea85a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -148,4 +149,9 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
public void checkVersion() {
// Does nothing. (Version is always compatible since it's in memory)
}
+
+ @Override
+ public void close() throws IOException {
+ // Does nothing.
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4c64449..1d5a6ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -99,7 +99,7 @@ public abstract class YarnConfigurationStore {
* Closes the configuration store, releasing any required resources.
* @throws IOException on failure to close
*/
- public void close() throws IOException {}
+ public abstract void close() throws IOException;
/**
* Logs the configuration change to backing store.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 61d27cc..229f680 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.ACL;
+import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
@@ -317,4 +318,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
private static <T> T unsafeCast(Object o) throws ClassCastException {
return (T)o;
}
+
+ @Override
+ public void close() throws IOException {
+ if (zkManager != null) {
+ zkManager.close();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org