You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2014/07/25 20:02:28 UTC
git commit: (TWILL-96) Make YARN unit-tests run on random port.
Repository: incubator-twill
Updated Branches:
refs/heads/master 378faf364 -> 6f43df38b
(TWILL-96) Make YARN unit-tests run on random port.
Signed-off-by: Terence Yim <te...@continuuity.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/6f43df38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/6f43df38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/6f43df38
Branch: refs/heads/master
Commit: 6f43df38bd5b50f655cd2c3a96a2486d42e77983
Parents: 378faf3
Author: Terence Yim <te...@continuuity.com>
Authored: Thu Jul 24 15:09:07 2014 -0700
Committer: Terence Yim <te...@continuuity.com>
Committed: Fri Jul 25 11:02:08 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/twill/internal/EnvKeys.java | 1 +
.../appmaster/ApplicationMasterMain.java | 18 +++++++++++++
.../apache/twill/yarn/YarnTwillPreparer.java | 4 ++-
.../org/apache/twill/yarn/YarnTestUtils.java | 27 ++++++++++----------
4 files changed, 36 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6f43df38/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
index 9bf6523..5fa42f8 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
@@ -41,6 +41,7 @@ public final class EnvKeys {
public static final String TWILL_LOG_KAFKA_ZK = "TWILL_LOG_KAFKA_ZK";
+ public static final String YARN_RM_SCHEDULER_ADDRESS = "YARN_RM_SCHEDULER_ADDRESS";
public static final String YARN_APP_ID = "YARN_APP_ID";
public static final String YARN_APP_ID_CLUSTER_TIME = "YARN_APP_ID_CLUSTER_TIME";
public static final String YARN_APP_ID_STR = "YARN_APP_ID_STR";
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6f43df38/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
index 880b5fd..a3619d7 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
@@ -64,11 +64,29 @@ public final class ApplicationMasterMain extends ServiceMain {
RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));
Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
+ setRMSchedulerAddress(conf);
Service service = new ApplicationMasterService(runId, zkClientService, twillSpec,
new VersionDetectYarnAMClientFactory(conf), createAppLocation(conf));
new ApplicationMasterMain(String.format("%s/%s/kafka", zkConnect, runId.getId())).doMain(zkClientService, service);
}
+ /**
+ * Optionally sets the RM scheduler address based on the environment variable if it is not set in the cluster config.
+ */
+ private static void setRMSchedulerAddress(Configuration conf) {
+ String schedulerAddress = System.getenv(EnvKeys.YARN_RM_SCHEDULER_ADDRESS);
+ if (schedulerAddress == null) {
+ return;
+ }
+
+ // If the RM scheduler address is not in the config or it's from yarn-default.xml,
+ // replace it with the one from the env, which is the same as the one client connected to.
+ String[] sources = conf.getPropertySources(YarnConfiguration.RM_SCHEDULER_ADDRESS);
+ if (sources == null || sources.length == 0 || "yarn-default.xml".equals(sources[sources.length - 1])) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, schedulerAddress);
+ }
+ }
+
@Override
protected String getHostname() {
try {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6f43df38/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 2ed2de0..36b4f0b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -287,7 +287,9 @@ final class YarnTwillPreparer implements TwillPreparer {
.put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
.put(EnvKeys.TWILL_RUN_ID, runId.getId())
.put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
- .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName()).build(),
+ .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS))
+ .build(),
localFiles.values(), credentials
)
.noResources()
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6f43df38/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
index bbeb5a2..bdf97e6 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
@@ -83,27 +83,28 @@ public final class YarnTestUtils {
fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, folder.getAbsolutePath());
dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();
- config = new YarnConfiguration(dfsCluster.getFileSystem().getConf());
+ Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());
if (YarnUtils.isHadoop20()) {
- config.set("yarn.resourcemanager.scheduler.class",
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
+ conf.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
} else {
- config.set("yarn.resourcemanager.scheduler.class",
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
- config.set("yarn.scheduler.capacity.resource-calculator",
- "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
+ conf.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
+ conf.set("yarn.scheduler.capacity.resource-calculator",
+ "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
}
- config.set("yarn.minicluster.fixed.ports", "true");
- config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
- config.set("yarn.nodemanager.vmem-check-enabled", "false");
- config.set("yarn.scheduler.minimum-allocation-mb", "128");
- config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");
+ conf.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
+ conf.set("yarn.nodemanager.vmem-check-enabled", "false");
+ conf.set("yarn.scheduler.minimum-allocation-mb", "128");
+ conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");
cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
- cluster.init(config);
+ cluster.init(conf);
cluster.start();
+ config = new YarnConfiguration(cluster.getConfig());
+
runnerService = createTwillRunnerService();
runnerService.startAndWait();
}