You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2018/11/29 01:42:22 UTC
hadoop git commit: YARN-9067. YARN Resource Manager is running OOM
because of leak of Configuration Object. Contributed by Eric Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk fe7dab8ef -> efc4d91cb
YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efc4d91c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efc4d91c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efc4d91c
Branch: refs/heads/trunk
Commit: efc4d91cbeab8a13f6d61cb0e56443adb2d77559
Parents: fe7dab8
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 29 09:34:14 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 29 09:34:14 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/service/webapp/ApiServer.java | 209 +++++++++++--------
.../hadoop/yarn/service/ServiceClientTest.java | 2 +-
.../yarn/service/client/ServiceClient.java | 1 +
3 files changed, 126 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index db831ba..88aeefd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -118,10 +118,13 @@ public class ApiServer {
@Override
public Void run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.actionBuild(service);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionBuild(service);
+ } finally {
+ sc.close();
+ }
return null;
}
});
@@ -133,11 +136,14 @@ public class ApiServer {
@Override
public ApplicationId run() throws IOException, YarnException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId applicationId = sc.actionCreate(service);
- sc.close();
- return applicationId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId applicationId = sc.actionCreate(service);
+ return applicationId;
+ } finally {
+ sc.close();
+ }
}
});
serviceStatus.setDiagnostics("Application ID: " + applicationId);
@@ -245,29 +251,32 @@ public class ApiServer {
public Integer run() throws Exception {
int result = 0;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Exception stopException = null;
try {
- result = sc.actionStop(appName, destroy);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully stopped service {}", appName);
- }
- } catch (Exception e) {
- LOG.info("Got exception stopping service", e);
- stopException = e;
- }
- if (destroy) {
- result = sc.actionDestroy(appName);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully deleted service {}", appName);
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Exception stopException = null;
+ try {
+ result = sc.actionStop(appName, destroy);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ } catch (Exception e) {
+ LOG.info("Got exception stopping service", e);
+ stopException = e;
}
- } else {
- if (stopException != null) {
- throw stopException;
+ if (destroy) {
+ result = sc.actionDestroy(appName);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully deleted service {}", appName);
+ }
+ } else {
+ if (stopException != null) {
+ throw stopException;
+ }
}
+ } finally {
+ sc.close();
}
- sc.close();
return result;
}
});
@@ -378,13 +387,16 @@ public class ApiServer {
@Override
public Map<String, Long> run() throws YarnException, IOException {
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Map<String, Long> original = sc.flexByRestService(appName,
- Collections.singletonMap(componentName,
- component.getNumberOfContainers()));
- sc.close();
- return original;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Map<String, Long> original = sc.flexByRestService(appName,
+ Collections.singletonMap(componentName,
+ component.getNumberOfContainers()));
+ return original;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -632,12 +644,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionFlex(appName, componentCountStrings);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionFlex(appName, componentCountStrings);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -658,12 +673,15 @@ public class ApiServer {
@Override
public String run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- String newLifeTime = sc.updateLifetime(appName,
- updateAppData.getLifetime());
- sc.close();
- return newLifeTime;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ String newLifeTime = sc.updateLifetime(appName,
+ updateAppData.getLifetime());
+ return newLifeTime;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -681,11 +699,14 @@ public class ApiServer {
@Override public ApplicationId run()
throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId appId = sc.actionStartAndGetId(appName);
- sc.close();
- return appId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId appId = sc.actionStartAndGetId(appName);
+ return appId;
+ } finally {
+ sc.close();
+ }
}
});
LOG.info("Successfully started service " + appName);
@@ -702,14 +723,17 @@ public class ApiServer {
ServiceStatus status = new ServiceStatus();
ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
- sc.actionUpgradeExpress(service);
- } else {
- sc.initiateUpgrade(service);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
+ sc.actionUpgradeExpress(service);
+ } else {
+ sc.initiateUpgrade(service);
+ }
+ } finally {
+ sc.close();
}
- sc.close();
return null;
});
LOG.info("Service {} version {} upgrade initialized", service.getName(),
@@ -724,11 +748,14 @@ public class ApiServer {
final UserGroupInformation ugi) throws IOException, InterruptedException {
int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- int exitCode = sc.actionCancelUpgrade(serviceName);
- sc.close();
- return exitCode;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ int exitCode = sc.actionCancelUpgrade(serviceName);
+ return exitCode;
+ } finally {
+ sc.close();
+ }
});
if (result == EXIT_SUCCESS) {
ServiceStatus status = new ServiceStatus();
@@ -793,10 +820,13 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
int result1;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result1 = sc.actionUpgrade(service, containers);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result1 = sc.actionUpgrade(service, containers);
+ } finally {
+ sc.close();
+ }
return result1;
});
}
@@ -815,12 +845,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionDecommissionInstances(appName, instances);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionDecommissionInstances(appName, instances);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -840,11 +873,14 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Service app1 = sc.getStatus(serviceName);
- sc.close();
- return app1;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Service app1 = sc.getStatus(serviceName);
+ return app1;
+ } finally {
+ sc.close();
+ }
});
}
@@ -855,12 +891,15 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> {
ComponentContainers[] result;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc.getContainers(serviceName, componentNames, version,
- containerStates);
- sc.close();
- return result;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc.getContainers(serviceName, componentNames, version,
+ containerStates);
+ return result;
+ } finally {
+ sc.close();
+ }
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index d022614..89366b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient {
public void forceStop() {
expectedInstances.clear();
- super.stop();
+ stop();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 713d890..5cdb8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (registryClient != null) {
registryClient.stop();
}
+ fs.getFileSystem().close();
super.serviceStop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org