You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/07/16 21:56:04 UTC
git commit: TEZ-299. Remove creation of local job conf as config
passed over in user payload for MR tasks. (hitesh)
Updated Branches:
refs/heads/master 6ede8fef7 -> 0e2090711
TEZ-299. Remove creation of local job conf as config passed over in user payload for MR tasks. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/0e209071
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/0e209071
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/0e209071
Branch: refs/heads/master
Commit: 0e2090711df41d920f61ba0647cfe453e8c40eb8
Parents: 6ede8fe
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Jul 16 12:55:50 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Jul 16 12:55:50 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/common/counters/Limits.java | 52 +++++++++++++++-----
.../apache/hadoop/mapred/YarnTezDagChild.java | 15 ++++--
.../tez/mapreduce/task/MRRuntimeTask.java | 7 ---
3 files changed, 50 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
index 71b4d1b..aacce87 100644
--- a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
+++ b/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
@@ -25,19 +25,34 @@ import org.apache.tez.common.TezJobConfig;
@InterfaceAudience.Private
public class Limits {
- static final Configuration conf = new Configuration();
- public static final int GROUP_NAME_MAX =
- conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
- TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
- public static final int COUNTER_NAME_MAX =
- conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY,
- TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
- public static final int GROUPS_MAX =
- conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY,
- TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
- public static final int COUNTERS_MAX =
- conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
- COUNTERS_MAX_DEFAULT);
+ private static Configuration conf = null;
+ private static int GROUP_NAME_MAX;
+ private static int COUNTER_NAME_MAX;
+ private static int GROUPS_MAX;
+ private static int COUNTERS_MAX;
+ private static boolean initialized = false;
+
+ private static synchronized void ensureInitialized() {
+ if (initialized) {
+ return;
+ }
+ if (conf == null) {
+ conf = new Configuration();
+ }
+ GROUP_NAME_MAX =
+ conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
+ TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
+ COUNTER_NAME_MAX =
+ conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY,
+ TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
+ GROUPS_MAX =
+ conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY,
+ TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
+ COUNTERS_MAX =
+ conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
+ COUNTERS_MAX_DEFAULT);
+ initialized = true;
+ }
private int totalCounters;
private LimitExceededException firstViolation;
@@ -47,14 +62,17 @@ public class Limits {
}
public static String filterCounterName(String name) {
+ ensureInitialized();
return filterName(name, COUNTER_NAME_MAX);
}
public static String filterGroupName(String name) {
+ ensureInitialized();
return filterName(name, GROUP_NAME_MAX);
}
public synchronized void checkCounters(int size) {
+ ensureInitialized();
if (firstViolation != null) {
throw new LimitExceededException(firstViolation);
}
@@ -71,6 +89,7 @@ public class Limits {
}
public synchronized void checkGroups(int size) {
+ ensureInitialized();
if (firstViolation != null) {
throw new LimitExceededException(firstViolation);
}
@@ -83,4 +102,11 @@ public class Limits {
public synchronized LimitExceededException violation() {
return firstViolation;
}
+
+ public synchronized static void setConfiguration(Configuration conf) {
+ if (Limits.conf == null && conf != null) {
+ Limits.conf = conf;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 4ff68af..bad68a9 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -56,6 +56,7 @@ import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.counters.Limits;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
@@ -79,12 +80,15 @@ public class YarnTezDagChild {
public static void main(String[] args) throws Throwable {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- LOG.debug("Child starting");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Child starting");
+ }
final Configuration defaultConf = new Configuration();
// Security settings will be loaded based on core-site and core-default. Don't
// depend on the jobConf for this.
UserGroupInformation.setConfiguration(defaultConf);
+ Limits.setConfiguration(defaultConf);
String host = args[0];
int port = Integer.parseInt(args[1]);
@@ -101,9 +105,12 @@ public class YarnTezDagChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
- LOG.info("Executing with tokens:");
- for (Token<?> token: credentials.getAllTokens()) {
- LOG.info(token);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Executing with tokens:");
+ for (Token<?> token: credentials.getAllTokens()) {
+ LOG.info(token);
+ }
}
// Create TaskUmbilicalProtocol as actual task owner.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 1a3a5d4..3e6a95a 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -209,13 +209,6 @@ public class MRRuntimeTask extends RuntimeTask {
// Set up the DistributedCache related configs
setupDistributedCacheConfig(job);
- // Overwrite the localized task jobconf which is linked to in the current
- // work-dir.
- Path localTaskFile = new Path(
- job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
- MRJobConfig.JOB_CONF_FILE);
- writeLocalJobFile(localTaskFile, job);
-
task.setConf(job);
}