You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/04/24 02:26:00 UTC
[06/50] [abbrv] tez git commit: EZ-1482. Fix memory issues for Local
Mode running concurrent tasks (pramachandran)
EZ-1482. Fix memory issues for Local Mode running concurrent tasks (pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19378d5c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19378d5c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19378d5c
Branch: refs/heads/TEZ-2003
Commit: 19378d5c2d79006b19140f2c5b9740d04942a3b8
Parents: 11b5843
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Wed Apr 15 21:48:07 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Wed Apr 15 21:48:07 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/launcher/LocalContainerLauncher.java | 10 +++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 605577e..56f6ea9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
and use MR configs correctly
TEZ-2318. Tez UI: source and sink page is broken as they are not populated.
http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7f5a7b1..5c8aab6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -90,6 +90,7 @@ public class LocalContainerLauncher extends AbstractService implements
private final String workingDirectory;
private final Map<String, String> localEnv = new HashMap<String, String>();
private final ExecutionContext executionContext;
+ private int numExecutors;
private final ConcurrentHashMap<ContainerId, ListenableFuture<TezChild.ContainerExecutionResult>>
runningContainers =
@@ -122,7 +123,7 @@ public class LocalContainerLauncher extends AbstractService implements
@Override
public synchronized void serviceInit(Configuration conf) {
- int numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
+ numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT);
Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor");
ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors,
@@ -328,11 +329,14 @@ public class LocalContainerLauncher extends AbstractService implements
containerEnv.putAll(localEnv);
containerEnv.put(Environment.USER.name(), context.getUser());
- // TODO TEZ-1482. Control the memory available based on number of executors
+ long memAvailable;
+ synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
+ memAvailable = Runtime.getRuntime().maxMemory() / numExecutors;
+ }
TezChild tezChild =
TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials,
- Runtime.getRuntime().maxMemory(), context.getUser());
+ memAvailable, context.getUser());
tezChild.setUmbilical(tezTaskUmbilicalProtocol);
return tezChild;
}