You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2018/03/16 21:15:18 UTC
[2/7] twill git commit: (TWILL-254) Update to use
ContainerId.fromString
(TWILL-254) Update to use ContainerId.fromString
This closes #65 on Github.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/36a4f118
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/36a4f118
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/36a4f118
Branch: refs/heads/branch-0.12.0
Commit: 36a4f1185ed41a28a5b30aeb836571beb7ffccb8
Parents: 1e97c74
Author: Clay Baenziger <cb...@bloomberg.net>
Authored: Fri Feb 2 22:35:37 2018 -0500
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Mar 16 14:09:58 2018 -0700
----------------------------------------------------------------------
pom.xml | 51 ++++++++++++++++++--
.../internal/yarn/Hadoop20YarnAMClient.java | 7 +++
.../internal/yarn/Hadoop21YarnAMClient.java | 7 +++
.../internal/yarn/Hadoop22YarnAMClient.java | 2 +-
.../internal/yarn/Hadoop26YarnAMClient.java | 42 ++++++++++++++++
.../internal/yarn/AbstractYarnAMClient.java | 12 +++--
.../yarn/VersionDetectYarnAMClientFactory.java | 10 +++-
.../apache/twill/internal/yarn/YarnUtils.java | 13 ++++-
8 files changed, 132 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2ca9bb4..656d944 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,7 +171,6 @@
<junit.version>4.11</junit.version>
<jopt-simple.version>3.2</jopt-simple.version>
<commons-compress.version>1.5</commons-compress.version>
- <hadoop.version>[2.0.2-alpha,2.3.0]</hadoop.version>
<hadoop20.output.dir>target/hadoop20-classes</hadoop20.output.dir>
</properties>
@@ -538,9 +537,6 @@
<properties>
<hadoop.version>2.3.0</hadoop.version>
</properties>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<build>
<plugins>
<plugin>
@@ -723,6 +719,53 @@
</build>
</profile>
<profile>
+ <id>hadoop-2.6</id>
+ <properties>
+ <hadoop.version>2.6.5</hadoop.version>
+ </properties>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop21</source>
+ <source>src/main/hadoop22</source>
+ <source>src/main/hadoop23</source>
+ <source>src/main/hadoop26</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-2.0</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop20</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>java8-test</id>
<modules>
<module>twill-java8-test</module>
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
index a990cc0..67bef3e 100644
--- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
+++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
@@ -26,11 +26,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
import org.apache.twill.internal.yarn.ports.AMRMClient;
import org.apache.twill.internal.yarn.ports.AMRMClientImpl;
@@ -72,6 +74,11 @@ public final class Hadoop20YarnAMClient extends AbstractYarnAMClient<AMRMClient.
}
@Override
+ protected ContainerId containerIdLookup(String containerIdStr) {
+ return (ConverterUtils.toContainerId(containerIdStr));
+ }
+
+ @Override
protected void startUp() throws Exception {
Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");
Preconditions.checkNotNull(trackerUrl, "Tracker URL not set.");
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
index 82f428a..f349b4e 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
@@ -27,11 +27,13 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +59,11 @@ public class Hadoop21YarnAMClient extends AbstractYarnAMClient<AMRMClient.Contai
};
}
+ @Override
+ protected ContainerId containerIdLookup(String containerIdStr) {
+ return (ConverterUtils.toContainerId(containerIdStr));
+ }
+
protected final AMRMClient<AMRMClient.ContainerRequest> amrmClient;
protected final Hadoop21YarnNMClient nmClient;
protected Resource maxCapability;
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java b/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java
index 3ee99e8..c1bb5fd 100644
--- a/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java
+++ b/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java
@@ -26,7 +26,7 @@ import java.util.List;
/**
* Wrapper class for AMRMClient for Hadoop version 2.2 or greater.
*/
-public final class Hadoop22YarnAMClient extends Hadoop21YarnAMClient {
+public class Hadoop22YarnAMClient extends Hadoop21YarnAMClient {
private static final Logger LOG = LoggerFactory.getLogger(Hadoop22YarnAMClient.class);
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java b/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java
new file mode 100644
index 0000000..372c422
--- /dev/null
+++ b/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * Wrapper class for AMRMClient for Hadoop version 2.6 or greater.
+ */
+public final class Hadoop26YarnAMClient extends Hadoop22YarnAMClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop26YarnAMClient.class);
+
+ public Hadoop26YarnAMClient(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ protected final ContainerId containerIdLookup(String containerIdStr) {
+ return (ContainerId.fromString(containerIdStr));
+ }
+}
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java
index e8f3b2b..b4b50cd 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java
@@ -27,7 +27,6 @@ import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.twill.internal.ProcessLauncher;
import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
import org.slf4j.Logger;
@@ -80,7 +79,7 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem
String masterContainerId = System.getenv().get(containerIdEnvName);
Preconditions.checkArgument(masterContainerId != null,
"Missing %s from environment", containerIdEnvName);
- this.containerId = ConverterUtils.toContainerId(masterContainerId);
+ this.containerId = containerIdLookup(masterContainerId);
this.inflightRequests = ArrayListMultimap.create();
this.pendingRequests = ArrayListMultimap.create();
this.pendingRemoves = Lists.newLinkedList();
@@ -89,7 +88,6 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem
this.blacklistedResources = Lists.newArrayList();
}
-
@Override
public final ContainerId getContainerId() {
return containerId;
@@ -227,6 +225,14 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem
}
/**
+ * Returns the ContainerId given a container ID string
+ *
+ * @param containerIdStr the container ID string to lookup
+ * @return A {@link ContainerId} instance representing the result.
+ */
+ protected abstract ContainerId containerIdLookup(String containerIdStr);
+
+ /**
* Adjusts the given resource capability to fit in the cluster limit.
*
* @param capability The capability to be adjusted.
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
index 943efb8..c6ab02b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
@@ -48,11 +48,17 @@ public final class VersionDetectYarnAMClientFactory implements YarnAMClientFacto
clzName = getClass().getPackage().getName() + ".Hadoop21YarnAMClient";
clz = (Class<YarnAMClient>) Class.forName(clzName);
break;
- default:
- // Uses hadoop-2.2 or above class
+ case HADOOP_22:
+ case HADOOP_23:
+ // Uses hadoop-2.2 class
clzName = getClass().getPackage().getName() + ".Hadoop22YarnAMClient";
clz = (Class<YarnAMClient>) Class.forName(clzName);
break;
+ default:
+ // Uses hadoop-2.6 or above class
+ clzName = getClass().getPackage().getName() + ".Hadoop26YarnAMClient";
+ clz = (Class<YarnAMClient>) Class.forName(clzName);
+ break;
}
return clz.getConstructor(Configuration.class).newInstance(conf);
http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
index c0aeb0c..d7e6eb0 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
@@ -68,7 +68,8 @@ public class YarnUtils {
HADOOP_20,
HADOOP_21,
HADOOP_22,
- HADOOP_23
+ HADOOP_23,
+ HADOOP_26
}
private static final Logger LOG = LoggerFactory.getLogger(YarnUtils.class);
@@ -263,7 +264,15 @@ public class YarnUtils {
Class.forName("org.apache.hadoop.yarn.client.cli.LogsCLI");
try {
Class.forName("org.apache.hadoop.yarn.conf.HAUtil");
- HADOOP_VERSION.set(HadoopVersions.HADOOP_23);
+ try {
+ Class[] args = new Class[1];
+ args[0] = String.class;
+ // see if we have a org.apache.hadoop.yarn.api.records.ContainerId.fromString() method
+ Class.forName("org.apache.hadoop.yarn.api.records.ContainerId").getMethod("fromString", args);
+ HADOOP_VERSION.set(HadoopVersions.HADOOP_26);
+ } catch (NoSuchMethodException e) {
+ HADOOP_VERSION.set(HadoopVersions.HADOOP_23);
+ }
} catch (ClassNotFoundException e) {
HADOOP_VERSION.set(HadoopVersions.HADOOP_22);
}