You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2016/12/27 22:14:36 UTC
hadoop git commit: YARN-5257. Fix unreleased resources and null
dereferences (yufeigu via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1bbd02327 -> 9262797e8
YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9262797e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9262797e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9262797e
Branch: refs/heads/trunk
Commit: 9262797e86453fc04b7ca3710b73b21fcdf9e6b4
Parents: 1bbd023
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Dec 27 14:14:08 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Dec 27 14:14:08 2016 -0800
----------------------------------------------------------------------
.../unmanagedamlauncher/UnmanagedAMLauncher.java | 10 +++++-----
.../org/apache/hadoop/yarn/client/cli/TopCLI.java | 15 ++++++++-------
.../yarn/client/api/impl/TimelineClientImpl.java | 2 +-
.../java/org/apache/hadoop/yarn/state/Graph.java | 2 +-
.../hadoop/yarn/state/VisualizeStateMachine.java | 4 +++-
.../hadoop/yarn/util/ProcfsBasedProcessTree.java | 5 ++++-
.../application/ApplicationImpl.java | 2 +-
.../containermanager/container/ContainerImpl.java | 2 +-
.../linux/resources/TrafficController.java | 13 +++++++------
.../localizer/LocalizedResource.java | 2 +-
.../localizer/ResourceLocalizationService.java | 1 +
11 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
index b9c7e80..030a730 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
@@ -192,11 +192,11 @@ public class UnmanagedAMLauncher {
throw new RuntimeException(ex);
}
tokenFile.deleteOnExit();
- DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile,
- true));
- credentials.writeTokenStorageToStream(os);
- os.close();
-
+ try (DataOutputStream os = new DataOutputStream(
+ new FileOutputStream(tokenFile, true))) {
+ credentials.writeTokenStorageToStream(os);
+ }
+
Map<String, String> env = System.getenv();
ArrayList<String> envAMList = new ArrayList<String>();
boolean setClasspath = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index f99dd48..cf7b747 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -774,13 +774,14 @@ public class TopCLI extends YarnCLI {
private JSONObject getJSONObject(URLConnection conn)
throws IOException, JSONException {
- InputStream in = conn.getInputStream();
- String encoding = conn.getContentEncoding();
- encoding = encoding == null ? "UTF-8" : encoding;
- String body = IOUtils.toString(in, encoding);
- JSONObject obj = new JSONObject(body);
- JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
- return clusterInfo;
+ try(InputStream in = conn.getInputStream()) {
+ String encoding = conn.getContentEncoding();
+ encoding = encoding == null ? "UTF-8" : encoding;
+ String body = IOUtils.toString(in, encoding);
+ JSONObject obj = new JSONObject(body);
+ JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
+ return clusterInfo;
+ }
}
private URL getClusterUrl() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index b275a94..4506c48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -805,7 +805,7 @@ public class TimelineClientImpl extends TimelineClient {
error.getErrorCode());
}
}
- } else if (type.equals(DOMAIN_DATA_TYPE)) {
+ } else if (type.equals(DOMAIN_DATA_TYPE) && domains != null) {
boolean hasError = false;
for (TimelineDomain domain : domains.getDomains()) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
index 390deb2..6fb9119 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
@@ -189,7 +189,7 @@ public class Graph {
public void save(String filepath) throws IOException {
try (OutputStreamWriter fout = new OutputStreamWriter(
- new FileOutputStream(filepath), Charset.forName("UTF-8"));) {
+ new FileOutputStream(filepath), Charset.forName("UTF-8"))) {
fout.write(generateGraphViz());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
index 26e93c6..1620397 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
@@ -49,7 +49,9 @@ public class VisualizeStateMachine {
if (gname.endsWith("Impl")) {
gname = gname.substring(0, gname.length()-4);
}
- ret.addSubGraph(factory.generateStateGraph(gname));
+ if (ret != null) {
+ ret.addSubGraph(factory.generateStateGraph(gname));
+ }
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 29bc277..bb2a77f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -803,7 +803,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
if (LOG.isDebugEnabled()) {
LOG.debug("MemInfo : " + key + " : Value : " + value);
}
- memoryMappingInfo.setMemInfo(key, value);
+
+ if (memoryMappingInfo != null) {
+ memoryMappingInfo.setMemInfo(key, value);
+ }
}
} catch (Throwable t) {
LOG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index b9197c2..112b43a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -603,7 +603,7 @@ public class ApplicationImpl implements Application {
} catch (InvalidStateTransitionException e) {
LOG.warn("Can't handle this event at current state", e);
}
- if (oldState != newState) {
+ if (newState != null && oldState != newState) {
LOG.info("Application " + applicationID + " transitioned from "
+ oldState + " to " + newState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 4a6be32..cae30cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1680,7 +1680,7 @@ public class ContainerImpl implements Container {
+ oldState + "], eventType: [" + event.getType() + "]," +
" container: [" + containerID + "]", e);
}
- if (oldState != newState) {
+ if (newState != null && oldState != newState) {
LOG.info("Container " + containerID + " transitioned from "
+ oldState
+ " to " + newState);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
index f1468fb..4698eb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
@@ -626,15 +626,16 @@ import java.util.regex.Pattern;
try {
File tcCmds = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_SUFFIX, new
File(tmpDirPath));
- Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds),
- "UTF-8");
- PrintWriter printWriter = new PrintWriter(writer);
- for (String command : commands) {
- printWriter.println(command);
+ try (
+ Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds),
+ "UTF-8");
+ PrintWriter printWriter = new PrintWriter(writer)) {
+ for (String command : commands) {
+ printWriter.println(command);
+ }
}
- printWriter.close();
operation.appendArgs(tcCmds.getAbsolutePath());
return operation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
index 6d67cac..98f4e21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
@@ -200,7 +200,7 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
} catch (InvalidStateTransitionException e) {
LOG.warn("Can't handle this event at current state", e);
}
- if (oldState != newState) {
+ if (newState != null && oldState != newState) {
if (LOG.isDebugEnabled()) {
LOG.debug("Resource " + resourcePath + (localPath != null ?
"(->" + localPath + ")": "") + " transitioned from " + oldState
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index e4d74f6..37473e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1051,6 +1051,7 @@ public class ResourceLocalizationService extends CompositeService
LOG.error(
"Got exception in parsing URL of LocalResource:"
+ rsrc.getResource(), e);
+ continue;
}
LocalizerResourceRequestEvent assoc = scheduled.get(req);
if (assoc == null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org