You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2016/12/16 19:51:04 UTC
[01/14] hadoop git commit: YARN-5999. AMRMClientAsync will stop if
any exceptions thrown on allocate call. Contributed by Jian He
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 f5e0bd30f -> f121645c4
YARN-5999. AMRMClientAsync will stop if any exceptions thrown on allocate call. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64a2d5be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64a2d5be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64a2d5be
Branch: refs/heads/YARN-5734
Commit: 64a2d5be91a7f344aadf820481c542c967bc46a8
Parents: f5e0bd3
Author: Xuan <xg...@apache.org>
Authored: Wed Dec 14 14:33:23 2016 -0800
Committer: Xuan <xg...@apache.org>
Committed: Wed Dec 14 14:33:23 2016 -0800
----------------------------------------------------------------------
.../api/async/impl/AMRMClientAsyncImpl.java | 30 ++++++++------------
.../api/async/impl/TestAMRMClientAsync.java | 2 +-
2 files changed, 13 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64a2d5be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index bc6cadd..3dd53d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -61,7 +61,7 @@ extends AMRMClientAsync<T> {
private final HeartbeatThread heartbeatThread;
private final CallbackHandlerThread handlerThread;
- private final BlockingQueue<AllocateResponse> responseQueue;
+ private final BlockingQueue<Object> responseQueue;
private final Object unregisterHeartbeatLock = new Object();
@@ -70,8 +70,6 @@ extends AMRMClientAsync<T> {
private volatile String collectorAddr;
- private volatile Throwable savedException;
-
/**
*
* @param intervalMs heartbeat interval in milliseconds between AM and RM
@@ -90,7 +88,6 @@ extends AMRMClientAsync<T> {
handlerThread = new CallbackHandlerThread();
responseQueue = new LinkedBlockingQueue<>();
keepRunning = true;
- savedException = null;
}
/**
@@ -111,9 +108,8 @@ extends AMRMClientAsync<T> {
super(client, intervalMs, callbackHandler);
heartbeatThread = new HeartbeatThread();
handlerThread = new CallbackHandlerThread();
- responseQueue = new LinkedBlockingQueue<AllocateResponse>();
+ responseQueue = new LinkedBlockingQueue<Object>();
keepRunning = true;
- savedException = null;
}
@Override
@@ -265,7 +261,7 @@ extends AMRMClientAsync<T> {
public void run() {
while (true) {
- AllocateResponse response = null;
+ Object response = null;
// synchronization ensures we don't send heartbeats after unregistering
synchronized (unregisterHeartbeatLock) {
if (!keepRunning) {
@@ -280,10 +276,7 @@ extends AMRMClientAsync<T> {
return;
} catch (Throwable ex) {
LOG.error("Exception on heartbeat", ex);
- savedException = ex;
- // interrupt handler thread in case it waiting on the queue
- handlerThread.interrupt();
- return;
+ response = ex;
}
if (response != null) {
while (true) {
@@ -316,19 +309,20 @@ extends AMRMClientAsync<T> {
return;
}
try {
- AllocateResponse response;
- if(savedException != null) {
- LOG.error("Stopping callback due to: ", savedException);
- handler.onError(savedException);
- return;
- }
+ Object object;
try {
- response = responseQueue.take();
+ object = responseQueue.take();
} catch (InterruptedException ex) {
LOG.info("Interrupted while waiting for queue", ex);
continue;
}
+ if (object instanceof Throwable) {
+ progress = handler.getProgress();
+ handler.onError((Throwable) object);
+ continue;
+ }
+ AllocateResponse response = (AllocateResponse) object;
String collectorAddress = response.getCollectorAddr();
TimelineClient timelineClient = client.getRegisteredTimelineClient();
if (timelineClient != null && collectorAddress != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64a2d5be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
index dac82e4..ba38340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
@@ -213,7 +213,7 @@ public class TestAMRMClientAsync {
asyncClient.stop();
// stopping should have joined all threads and completed all callbacks
- Assert.assertTrue(callbackHandler.callbackCount == 0);
+ Assert.assertTrue(callbackHandler.callbackCount > 0);
}
@Test (timeout = 10000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/14] hadoop git commit: HADOOP-11804. Shaded Hadoop client
artifacts and minicluster. Contributed by Sean Busbey.
Posted by xg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
new file mode 100644
index 0000000..0394cae
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -0,0 +1,338 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+<parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project-dist</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project-dist</relativePath>
+</parent>
+ <artifactId>hadoop-client</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
+ <name>Apache Hadoop Client Aggregator</name>
+
+<properties>
+ <hadoop.component>client</hadoop.component>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jetty</groupId>
+ <artifactId>org.eclipse.jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jdt</groupId>
+ <artifactId>core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.kosmosfs</groupId>
+ <artifactId>kfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <!-- No slf4j backends for downstream clients -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <!-- No slf4j backends for downstream clients -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <!-- No slf4j backends for downstream clients -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <!-- No slf4j backends for downstream clients -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
new file mode 100644
index 0000000..eea4c8f
--- /dev/null
+++ b/hadoop-client-modules/pom.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-client-modules</artifactId>
+ <packaging>pom</packaging>
+ <description>multi-module for Apache Hadoop client artifacts</description>
+ <name>Apache Hadoop Client Modules</name>
+
+ <modules>
+ <!-- Left as an empty artifact w/dep for compat -->
+ <module>hadoop-client</module>
+ <!-- Should be used at compile scope for access to IA.Public classes -->
+ <module>hadoop-client-api</module>
+ <!-- Should be used at runtime scope for remaining classes necessary for hadoop-client-api to function -->
+ <module>hadoop-client-runtime</module>
+ <!-- Should be used at test scope for those that need access to mini cluster that works with above api and runtime -->
+ <module>hadoop-client-minicluster</module>
+ <!-- Checks invariants above -->
+ <module>hadoop-client-check-invariants</module>
+ <module>hadoop-client-check-test-invariants</module>
+ <!-- Attempt to use the created libraries -->
+ <module>hadoop-client-integration-tests</module>
+ </modules>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
deleted file mode 100644
index dbd4fac..0000000
--- a/hadoop-client/pom.xml
+++ /dev/null
@@ -1,319 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-<parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha2-SNAPSHOT</version>
- <relativePath>../hadoop-project-dist</relativePath>
- </parent>
- <artifactId>hadoop-client</artifactId>
- <version>3.0.0-alpha2-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <description>Apache Hadoop Client</description>
- <name>Apache Hadoop Client</name>
-
-<properties>
- <hadoop.component>client</hadoop.component>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jetty</groupId>
- <artifactId>org.eclipse.jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jdt</groupId>
- <artifactId>core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.sf.kosmosfs</groupId>
- <artifactId>kfs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-app</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-nodemanager</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>jdk.tools</groupId>
- <artifactId>jdk.tools</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- </dependencies>
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 30e6472..ae8c4ce 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -51,6 +51,23 @@
<artifactId>hadoop-yarn-api</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-check-invariants</artifactId>
+ <type>pom</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-check-test-invariants</artifactId>
+ <type>pom</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-integration-tests</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index eae03a1..6a2bd73 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -54,6 +54,28 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <!-- shade conflicts with itself and our other maven internals
+ by including a combination of 3.x and 2.x
+ -->
+ <exclusion>
+ <groupId>org.apache.maven.shared</groupId>
+ <artifactId>maven-dependency-tree</artifactId>
+ </exclusion>
+ <!-- again, shade conflicts with itself on the transitives of this
+ dependency
+ -->
+ <exclusion>
+ <groupId>org.vafer</groupId>
+ <artifactId>jdependency</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
new file mode 100644
index 0000000..a54b5ba
--- /dev/null
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
@@ -0,0 +1,164 @@
+package org.apache.hadoop.maven.plugin.shade.resource;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+import java.io.BufferedReader;
+import org.apache.maven.plugins.shade.relocation.Relocator;
+import org.apache.maven.plugins.shade.resource.ResourceTransformer;
+import org.codehaus.plexus.util.IOUtil;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+
+/**
+ * Resources transformer that appends entries in META-INF/services resources
+ * into a single resource. For example, if there are several
+ * META-INF/services/org.apache.maven.project.ProjectBuilder resources spread
+ * across many JARs the individual entries will all be concatenated into a
+ * single META-INF/services/org.apache.maven.project.ProjectBuilder resource
+ * packaged into the resultant JAR produced by the shading process.
+ *
+ * From following sources, only needed until MSHADE-182 gets released
+ * * https://s.apache.org/vwjl (source in maven-shade-plugin repo)
+ * * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch
+ *
+ * Has been reformatted according to Hadoop checkstyle rules and modified
+ * to meet Hadoop's threshold for Findbugs problems.
+ */
+public class ServicesResourceTransformer
+ implements ResourceTransformer {
+
+ private static final String SERVICES_PATH = "META-INF/services";
+
+ private Map<String, ServiceStream> serviceEntries = new HashMap<>();
+
+ private List<Relocator> relocators;
+
+ public boolean canTransformResource(String resource) {
+ if (resource.startsWith(SERVICES_PATH)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public void processResource(String resource, InputStream is,
+ List<Relocator> relocatorz) throws IOException {
+ ServiceStream out = serviceEntries.get(resource);
+ if (out == null) {
+ out = new ServiceStream();
+ serviceEntries.put(resource, out);
+ }
+
+ out.append(is);
+ is.close();
+
+ if (this.relocators == null) {
+ this.relocators = relocatorz;
+ }
+ }
+
+ public boolean hasTransformedResource() {
+ return serviceEntries.size() > 0;
+ }
+
+ public void modifyOutputStream(JarOutputStream jos)
+ throws IOException {
+ for (Map.Entry<String, ServiceStream> entry : serviceEntries.entrySet()) {
+ String key = entry.getKey();
+ ServiceStream data = entry.getValue();
+
+ if (relocators != null) {
+ key = key.substring(SERVICES_PATH.length() + 1);
+ for (Relocator relocator : relocators) {
+ if (relocator.canRelocateClass(key)) {
+ key = relocator.relocateClass(key);
+ break;
+ }
+ }
+
+ key = SERVICES_PATH + '/' + key;
+ }
+
+ jos.putNextEntry(new JarEntry(key));
+
+ //read the content of service file for candidate classes for relocation
+ //presume everything is UTF8, because Findbugs barfs on default
+ //charset and this seems no worse a choice �\_(\u30c4)_/�
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos,
+ StandardCharsets.UTF_8));
+ InputStreamReader streamReader =
+ new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(streamReader);
+ String className;
+
+ while ((className = reader.readLine()) != null) {
+
+ if (relocators != null) {
+ for (Relocator relocator : relocators) {
+ //if the class can be relocated then relocate it
+ if (relocator.canRelocateClass(className)) {
+ className = relocator.applyToSourceContent(className);
+ break;
+ }
+ }
+ }
+
+ writer.println(className);
+ writer.flush();
+ }
+
+ reader.close();
+ data.reset();
+ }
+ }
+
+ static class ServiceStream extends ByteArrayOutputStream {
+
+ public ServiceStream() {
+ super(1024);
+ }
+
+ public void append(InputStream is)
+ throws IOException {
+ if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') {
+ write('\n');
+ }
+
+ IOUtil.copy(is, this);
+ }
+
+ public InputStream toInputStream() {
+ return new ByteArrayInputStream(buf, 0, count);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
new file mode 100644
index 0000000..571491c
--- /dev/null
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Resource handling plugins used internal to the Hadoop build.
+ * IA.Private (build structure encourages not using the actual annotations)
+ */
+package org.apache.hadoop.maven.plugin.shade.resource;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 9ec24ea..36172f2 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -27,6 +27,7 @@
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
+ <inceptionYear>2008</inceptionYear>
<properties>
<!-- Set the Release year during release -->
@@ -114,6 +115,7 @@
<maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
<maven-install-plugin.version>2.5.1</maven-install-plugin.version>
<maven-resources-plugin.version>2.6</maven-resources-plugin.version>
+ <maven-shade-plugin.version>2.4.3</maven-shade-plugin.version>
<maven-jar-plugin.version>2.5</maven-jar-plugin.version>
<maven-war-plugin.version>2.4</maven-war-plugin.version>
<maven-source-plugin.version>2.3</maven-source-plugin.version>
@@ -150,6 +152,44 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-modules</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-check-invariants</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-check-test-invariants</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-integration-tests</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${project.version}</version>
</dependency>
@@ -1245,6 +1285,40 @@
<pluginManagement>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>${build-helper-maven-plugin.version}</version>
+ </plugin>
+ <!-- We include the configuration for license-maven-plugin to correct
+ maven-shade-plugin generated poms because it's always the same. We
+ can't simply configure the plugin because we must ensure execution
+ happens in the package phase after the shade plugin runs.
+ -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>1.10</version>
+ <configuration>
+ <canUpdateCopyright>false</canUpdateCopyright>
+ <roots><root>${project.basedir}</root></roots>
+ </configuration>
+ <executions>
+ <execution>
+ <id>update-pom-license</id>
+ <goals>
+ <goal>update-file-header</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <licenseName>apache_v2</licenseName>
+ <includes>
+ <include>dependency-reduced-pom.xml</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>${maven-clean-plugin.version}</version>
</plugin>
@@ -1260,13 +1334,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.2</version>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>${build-helper-maven-plugin.version}</version>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1356,6 +1425,19 @@
<plugins>
<plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>${project.basedir}</directory>
+ <includes>
+ <include>dependency-reduced-pom.xml</include>
+ </includes>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8776534..9de7b36 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-stylus-skin.version>1.5</maven-stylus-skin.version>
<maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
- <maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
+ <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
<maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
<maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
<maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
@@ -125,8 +125,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<module>hadoop-mapreduce-project</module>
<module>hadoop-tools</module>
<module>hadoop-dist</module>
- <module>hadoop-client</module>
<module>hadoop-minicluster</module>
+ <module>hadoop-client-modules</module>
<module>hadoop-build-tools</module>
</modules>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/14] hadoop git commit: HADOOP-13831. Correct check for error code
to detect Azure Storage Throttling and provide retries. Contributed by Gaurav
Kanade
Posted by xg...@apache.org.
HADOOP-13831. Correct check for error code to detect Azure Storage Throttling and provide retries. Contributed by Gaurav Kanade
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f92913c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f92913c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f92913c3
Branch: refs/heads/YARN-5734
Commit: f92913c35bfda0d565606f9fb9a301ddd4105fd8
Parents: 70ca1f1
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Dec 15 12:35:08 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Dec 15 12:35:08 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/fs/azure/AzureNativeFileSystemStore.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f92913c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index ac6c514..d232a2d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
@@ -2511,8 +2512,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
try {
dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
} catch (StorageException se) {
- if (se.getErrorCode().equals(
- StorageErrorCode.SERVER_BUSY.toString())) {
+ if (se.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
int copyBlobMinBackoff = sessionConfiguration.getInt(
KEY_COPYBLOB_MIN_BACKOFF_INTERVAL,
DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL);
@@ -2541,8 +2541,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
waitForCopyToComplete(dstBlob, getInstrumentedContext());
safeDelete(srcBlob, lease);
} catch (StorageException e) {
- if (e.getErrorCode().equals(
- StorageErrorCode.SERVER_BUSY.toString())) {
+ if (e.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
LOG.warn("Rename: CopyBlob: StorageException: ServerBusy: Retry complete, will attempt client side copy for page blob");
InputStream ipStream = null;
OutputStream opStream = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/14] hadoop git commit: HDFS-11253. FileInputStream leak on
failure path in BlockSender. Contributed by Arpit Agarwal
Posted by xg...@apache.org.
HDFS-11253. FileInputStream leak on failure path in BlockSender. Contributed by Arpit Agarwal
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fce1915
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fce1915
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fce1915
Branch: refs/heads/YARN-5734
Commit: 6fce19154906cb271b637b46896a34e7701a7bac
Parents: ac0d402
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Dec 15 15:39:15 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Dec 15 15:39:15 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fce1915/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index d7aebd8..99597dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -407,6 +407,8 @@ class BlockSender implements java.io.Closeable {
blockIn, checksumIn, volumeRef, fileIoProvider);
} catch (IOException ioe) {
IOUtils.closeStream(this);
+ org.apache.commons.io.IOUtils.closeQuietly(blockIn);
+ org.apache.commons.io.IOUtils.closeQuietly(checksumIn);
throw ioe;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/14] hadoop git commit: This closes #119. (empty commit)
Posted by xg...@apache.org.
This closes #119. (empty commit)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f121645c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f121645c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f121645c
Branch: refs/heads/YARN-5734
Commit: f121645c4bdfdabfd6e4c40cd0c3dc8930372e17
Parents: a956390
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Dec 16 11:04:32 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Dec 16 11:04:32 2016 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/14] hadoop git commit: HDFS-11094. Send back HAState along with
NamespaceInfo during a versionRequest as an optional parameter. Contributed
by Eric Badger
Posted by xg...@apache.org.
HDFS-11094. Send back HAState along with NamespaceInfo during a versionRequest as an optional parameter. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac0d402c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac0d402c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac0d402c
Branch: refs/heads/YARN-5734
Commit: ac0d402c6a3af784aa8bdceecf3293c30760346f
Parents: 631f1da
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Dec 15 12:48:27 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Dec 15 14:53:53 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 72 +++++++++++++-------
.../hdfs/server/datanode/BPOfferService.java | 10 ++-
.../hdfs/server/datanode/BPServiceActor.java | 4 +-
.../hdfs/server/namenode/FSNamesystem.java | 8 ++-
.../hdfs/server/protocol/NamespaceInfo.java | 26 +++++++
.../src/main/proto/DatanodeProtocol.proto | 13 ----
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 15 ++++
.../server/datanode/TestBPOfferService.java | 32 +++++++++
.../hdfs/server/namenode/TestFSNamesystem.java | 21 ++++++
9 files changed, 158 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 4f6a04e..d97708f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeComm
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.FinalizeCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCommandProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto;
@@ -66,6 +65,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NamenodeCommandPro
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NamenodeRegistrationProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NamenodeRegistrationProto.NamenodeRoleProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NamespaceInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NNHAStatusHeartbeatProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.RecoveringBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.RemoteEditLogManifestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.RemoteEditLogProto;
@@ -337,7 +337,8 @@ public class PBHelper {
StorageInfoProto storage = info.getStorageInfo();
return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(),
info.getBlockPoolID(), storage.getCTime(), info.getBuildVersion(),
- info.getSoftwareVersion(), info.getCapabilities());
+ info.getSoftwareVersion(), info.getCapabilities(),
+ convert(info.getState()));
}
public static NamenodeCommand convert(NamenodeCommandProto cmd) {
@@ -743,43 +744,64 @@ public class PBHelper {
}
public static NamespaceInfoProto convert(NamespaceInfo info) {
- return NamespaceInfoProto.newBuilder()
- .setBlockPoolID(info.getBlockPoolID())
+ NamespaceInfoProto.Builder builder = NamespaceInfoProto.newBuilder();
+ builder.setBlockPoolID(info.getBlockPoolID())
.setBuildVersion(info.getBuildVersion())
.setUnused(0)
.setStorageInfo(PBHelper.convert((StorageInfo)info))
.setSoftwareVersion(info.getSoftwareVersion())
- .setCapabilities(info.getCapabilities())
- .build();
+ .setCapabilities(info.getCapabilities());
+ HAServiceState state = info.getState();
+ if(state != null) {
+ builder.setState(convert(info.getState()));
+ }
+ return builder.build();
}
- public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) {
- if (s == null) return null;
- switch (s.getState()) {
+ public static HAServiceState convert(NNHAStatusHeartbeatProto.State s) {
+ if (s == null) {
+ return null;
+ }
+ switch (s) {
case ACTIVE:
- return new NNHAStatusHeartbeat(HAServiceState.ACTIVE, s.getTxid());
+ return HAServiceState.ACTIVE;
case STANDBY:
- return new NNHAStatusHeartbeat(HAServiceState.STANDBY, s.getTxid());
+ return HAServiceState.STANDBY;
default:
- throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" + s.getState());
+ throw new IllegalArgumentException("Unexpected HAServiceStateProto:"
+ + s);
}
}
+ public static NNHAStatusHeartbeatProto.State convert(HAServiceState s) {
+ if (s == null) {
+ return null;
+ }
+ switch (s) {
+ case ACTIVE:
+ return NNHAStatusHeartbeatProto.State.ACTIVE;
+ case STANDBY:
+ return NNHAStatusHeartbeatProto.State.STANDBY;
+ default:
+ throw new IllegalArgumentException("Unexpected HAServiceState:"
+ + s);
+ }
+ }
+
+ public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) {
+ if (s == null) {
+ return null;
+ }
+ return new NNHAStatusHeartbeat(convert(s.getState()), s.getTxid());
+ }
+
public static NNHAStatusHeartbeatProto convert(NNHAStatusHeartbeat hb) {
- if (hb == null) return null;
- NNHAStatusHeartbeatProto.Builder builder =
- NNHAStatusHeartbeatProto.newBuilder();
- switch (hb.getState()) {
- case ACTIVE:
- builder.setState(NNHAStatusHeartbeatProto.State.ACTIVE);
- break;
- case STANDBY:
- builder.setState(NNHAStatusHeartbeatProto.State.STANDBY);
- break;
- default:
- throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" +
- hb.getState());
+ if (hb == null) {
+ return null;
}
+ NNHAStatusHeartbeatProto.Builder builder =
+ NNHAStatusHeartbeatProto.newBuilder();
+ builder.setState(convert(hb.getState()));
builder.setTxid(hb.getTxId());
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 00102eb..00e6b3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -307,8 +307,16 @@ class BPOfferService {
* verifies that this namespace matches (eg to prevent a misconfiguration
* where a StandbyNode from a different cluster is specified)
*/
- void verifyAndSetNamespaceInfo(NamespaceInfo nsInfo) throws IOException {
+ void verifyAndSetNamespaceInfo(BPServiceActor actor, NamespaceInfo nsInfo)
+ throws IOException {
writeLock();
+
+ if(nsInfo.getState() == HAServiceState.ACTIVE
+ && bpServiceToActive == null) {
+ LOG.info("Acknowledging ACTIVE Namenode during handshake" + actor);
+ bpServiceToActive = actor;
+ }
+
try {
if (this.bpNSInfo == null) {
this.bpNSInfo = nsInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index bb2b792..25ab3b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -269,11 +269,11 @@ class BPServiceActor implements Runnable {
// First phase of the handshake with NN - get the namespace
// info.
NamespaceInfo nsInfo = retrieveNamespaceInfo();
-
+
// Verify that this matches the other NN in this HA pair.
// This also initializes our block pool in the DN if we are
// the first NN connection for this BP.
- bpos.verifyAndSetNamespaceInfo(nsInfo);
+ bpos.verifyAndSetNamespaceInfo(this, nsInfo);
// Second phase of the handshake with the NN.
register(nsInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8a750a0..90fb924 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1594,7 +1594,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
NamespaceInfo unprotectedGetNamespaceInfo() {
return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(),
getClusterId(), getBlockPoolId(),
- getFSImage().getStorage().getCTime());
+ getFSImage().getStorage().getCTime(), getState());
}
/**
@@ -4531,12 +4531,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return 0;
}
}
-
+
@Metric
public int getBlockCapacity() {
return blockManager.getCapacity();
}
+ public HAServiceState getState() {
+ return haContext == null ? null : haContext.getState().getServiceState();
+ }
+
@Override // FSNamesystemMBean
public String getFSState() {
return isInSafeMode() ? "safeMode" : "Operational";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
index 90d0aac..66ce9ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
@@ -44,6 +45,7 @@ public class NamespaceInfo extends StorageInfo {
String blockPoolID = ""; // id of the block pool
String softwareVersion;
long capabilities;
+ HAServiceState state;
// only authoritative on the server-side to determine advertisement to
// clients. enum will update the supported values
@@ -88,6 +90,14 @@ public class NamespaceInfo extends StorageInfo {
CAPABILITIES_SUPPORTED);
}
+ public NamespaceInfo(int nsID, String clusterID, String bpID,
+ long cT, String buildVersion, String softwareVersion,
+ long capabilities, HAServiceState st) {
+ this(nsID, clusterID, bpID, cT, buildVersion, softwareVersion,
+ capabilities);
+ this.state = st;
+ }
+
// for use by server and/or client
public NamespaceInfo(int nsID, String clusterID, String bpID,
long cT, String buildVersion, String softwareVersion,
@@ -105,6 +115,13 @@ public class NamespaceInfo extends StorageInfo {
this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(),
VersionInfo.getVersion());
}
+
+ public NamespaceInfo(int nsID, String clusterID, String bpID,
+ long cT, HAServiceState st) {
+ this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(),
+ VersionInfo.getVersion());
+ this.state = st;
+ }
public long getCapabilities() {
return capabilities;
@@ -115,6 +132,11 @@ public class NamespaceInfo extends StorageInfo {
this.capabilities = capabilities;
}
+ @VisibleForTesting
+ public void setState(HAServiceState state) {
+ this.state = state;
+ }
+
public boolean isCapabilitySupported(Capability capability) {
Preconditions.checkArgument(capability != Capability.UNKNOWN,
"cannot test for unknown capability");
@@ -134,6 +156,10 @@ public class NamespaceInfo extends StorageInfo {
return softwareVersion;
}
+ public HAServiceState getState() {
+ return state;
+ }
+
@Override
public String toString(){
return super.toString() + ";bpid=" + blockPoolID;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
index 016eae2..7423b33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
@@ -209,19 +209,6 @@ message HeartbeatRequestProto {
}
/**
- * state - State the NN is in when returning response to the DN
- * txid - Highest transaction ID this NN has seen
- */
-message NNHAStatusHeartbeatProto {
- enum State {
- ACTIVE = 0;
- STANDBY = 1;
- }
- required State state = 1;
- required uint64 txid = 2;
-}
-
-/**
* cmds - Commands from namenode to datanode.
* haStatus - Status (from an HA perspective) of the NN sending this response
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
index 910e03b..e50883a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
@@ -32,6 +32,7 @@ option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "hdfs.proto";
+import "HAServiceProtocol.proto";
/**
* Block access token information
@@ -101,6 +102,7 @@ message NamespaceInfoProto {
required StorageInfoProto storageInfo = 4;// Node information
required string softwareVersion = 5; // Software version number (e.g. 2.0.0)
optional uint64 capabilities = 6 [default = 0]; // feature flags
+ optional NNHAStatusHeartbeatProto.State state = 7;
}
/**
@@ -201,3 +203,16 @@ message NamenodeRegistrationProto {
required StorageInfoProto storageInfo = 3; // Node information
optional NamenodeRoleProto role = 4 [default = NAMENODE]; // Namenode role
}
+
+/**
+ * state - State the NN is in when returning response to the DN
+ * txid - Highest transaction ID this NN has seen
+ */
+message NNHAStatusHeartbeatProto {
+ enum State {
+ ACTIVE = 0;
+ STANDBY = 1;
+ }
+ required State state = 1;
+ required uint64 txid = 2;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index 2d50c75..b7b8966 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
@@ -799,4 +800,35 @@ public class TestBPOfferService {
}
return -1;
}
+
+ /*
+ *
+ */
+ @Test
+ public void testNNHAStateUpdateFromVersionRequest() throws Exception {
+ final BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
+ Mockito.doReturn(true).when(mockDn).areHeartbeatsDisabledForTests();
+ BPServiceActor actor = bpos.getBPServiceActors().get(0);
+ bpos.start();
+ waitForInitialization(bpos);
+ // Should start with neither NN as active.
+ assertNull(bpos.getActiveNN());
+
+ // getNamespaceInfo() will not include HAServiceState
+ NamespaceInfo nsInfo = mockNN1.versionRequest();
+ bpos.verifyAndSetNamespaceInfo(actor, nsInfo);
+
+ assertNull(bpos.getActiveNN());
+
+ // Change mock so getNamespaceInfo() will include HAServiceState
+ Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0,
+ HAServiceState.ACTIVE)).when(mockNN1).versionRequest();
+
+ // Update the bpos NamespaceInfo
+ nsInfo = mockNN1.versionRequest();
+ bpos.verifyAndSetNamespaceInfo(actor, nsInfo);
+
+ assertNotNull(bpos.getActiveNN());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0d402c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index f02c679..6a0dd6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -33,6 +33,7 @@ import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
@@ -155,6 +157,25 @@ public class TestFSNamesystem {
}
@Test
+ public void testHAStateInNamespaceInfo() throws IOException {
+ Configuration conf = new Configuration();
+
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+ NNStorage nnStorage = Mockito.mock(NNStorage.class);
+ Mockito.when(fsImage.getStorage()).thenReturn(nnStorage);
+
+ FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage);
+ FSNamesystem fsn = Mockito.spy(fsNamesystem);
+ Mockito.when(fsn.getState()).thenReturn(
+ HAServiceProtocol.HAServiceState.ACTIVE);
+
+ NamespaceInfo nsInfo = fsn.unprotectedGetNamespaceInfo();
+ assertNotNull(nsInfo.getState());
+ }
+
+ @Test
public void testReset() throws Exception {
Configuration conf = new Configuration();
FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/14] hadoop git commit: HDFS-11249. Redundant toString() in
DFSConfigKeys.java. Contributed by Jagadesh Kiran N.
Posted by xg...@apache.org.
HDFS-11249. Redundant toString() in DFSConfigKeys.java. Contributed by Jagadesh Kiran N.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/36947f79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/36947f79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/36947f79
Branch: refs/heads/YARN-5734
Commit: 36947f79ba66bf8ce150813e5dc1d06c789f2b2a
Parents: 64a2d5b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Dec 16 00:33:27 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Dec 16 00:33:27 2016 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36947f79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index cffc4bd..80ceaaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -249,7 +249,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
need to update the literal here as well as TestDFSConfigKeys.
*/
public static final String DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT =
- "org.apache.hadoop.hdfs.web.AuthFilter".toString();
+ "org.apache.hadoop.hdfs.web.AuthFilter";
@Deprecated
public static final String DFS_WEBHDFS_USER_PATTERN_KEY =
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/14] hadoop git commit: HDFS-9911. TestDataNodeLifeline Fails
intermittently. Contributed by Yiqun Lin
Posted by xg...@apache.org.
HDFS-9911. TestDataNodeLifeline Fails intermittently. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9563906
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9563906
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9563906
Branch: refs/heads/YARN-5734
Commit: a95639068c99ebcaefe8b6c4268449d12a6577d6
Parents: 2273a74
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Dec 16 09:46:21 2016 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Dec 16 09:46:21 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9563906/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 25ab3b7..5294799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -1068,7 +1068,7 @@ class BPServiceActor implements Runnable {
volatile long nextHeartbeatTime = monotonicNow();
@VisibleForTesting
- volatile long nextLifelineTime = monotonicNow();
+ volatile long nextLifelineTime;
@VisibleForTesting
volatile long lastBlockReportTime = monotonicNow();
@@ -1091,6 +1091,7 @@ class BPServiceActor implements Runnable {
this.heartbeatIntervalMs = heartbeatIntervalMs;
this.lifelineIntervalMs = lifelineIntervalMs;
this.blockReportIntervalMs = blockReportIntervalMs;
+ scheduleNextLifeline(nextHeartbeatTime);
}
// This is useful to make sure NN gets Heartbeat before Blockreport
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/14] hadoop git commit: HDFS-11188. Change min supported DN and NN
versions back to 2.x. Contributed by Andrew Wang.
Posted by xg...@apache.org.
HDFS-11188. Change min supported DN and NN versions back to 2.x. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cee0c468
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cee0c468
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cee0c468
Branch: refs/heads/YARN-5734
Commit: cee0c468b0ebe2ae8727502620e345745ba11f77
Parents: aebb912
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Dec 15 16:48:50 2016 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Dec 15 16:48:50 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cee0c468/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 80ceaaf..15bb0bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -290,7 +290,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY = "dfs.namenode.max.extra.edits.segments.retained";
public static final int DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_DEFAULT = 10000; // 10k
public static final String DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY = "dfs.namenode.min.supported.datanode.version";
- public static final String DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT = "3.0.0-alpha1-SNAPSHOT";
+ public static final String DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT = "2.1.0-beta";
public static final String DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY = "dfs.namenode.edits.dir.minimum";
public static final int DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT = 1;
@@ -613,7 +613,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_IPC_DEFAULT_PORT = 9867;
public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_IPC_DEFAULT_PORT;
public static final String DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY = "dfs.datanode.min.supported.namenode.version";
- public static final String DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_DEFAULT = "3.0.0-alpha1-SNAPSHOT";
+ public static final String DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_DEFAULT = "2.1.0-beta";
public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY = "dfs.namenode.inode.attributes.provider.class";
public static final String DFS_DATANODE_BP_READY_TIMEOUT_KEY = "dfs.datanode.bp-ready.timeout";
public static final long DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT = 20;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/14] hadoop git commit: HADOOP-13709. Ability to clean up
subprocesses spawned by Shell when the process exits. Contributed by Eric
Badger
Posted by xg...@apache.org.
HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the process exits. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/631f1dae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/631f1dae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/631f1dae
Branch: refs/heads/YARN-5734
Commit: 631f1daee3507a1adbc68b937cca31c27dbe8d3d
Parents: 169bfc0
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Dec 15 20:52:40 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Dec 15 20:52:40 2016 +0000
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/util/Shell.java | 24 +++++++++
.../java/org/apache/hadoop/util/TestShell.java | 52 ++++++++++++++++++++
2 files changed, 76 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/631f1dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 5fc9869..83877b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -26,9 +26,11 @@ import java.io.InputStream;
import java.io.InterruptedIOException;
import java.nio.charset.Charset;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
@@ -48,6 +50,8 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class Shell {
+ private static final Map <Process, Object> CHILD_PROCESSES =
+ Collections.synchronizedMap(new WeakHashMap<Process, Object>());
public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
/**
@@ -916,6 +920,7 @@ public abstract class Shell {
} else {
process = builder.start();
}
+ CHILD_PROCESSES.put(process, null);
if (timeOutInterval > 0) {
timeOutTimer = new Timer("Shell command timeout");
@@ -1012,6 +1017,7 @@ public abstract class Shell {
LOG.warn("Error while closing the error stream", ioe);
}
process.destroy();
+ CHILD_PROCESSES.remove(process);
lastTime = Time.monotonicNow();
}
}
@@ -1310,4 +1316,22 @@ public abstract class Shell {
}
}
}
+
+ /**
+ * Static method to destroy all running <code>Shell</code> processes
+ * Iterates through a list of all currently running <code>Shell</code>
+ * processes and destroys them one by one. This method is thread safe and
+ * is intended to be used in a shutdown hook.
+ */
+ public static void destroyAllProcesses() {
+ synchronized (CHILD_PROCESSES) {
+ for (Process key : CHILD_PROCESSES.keySet()) {
+ Process process = key;
+ if (key != null) {
+ process.destroy();
+ }
+ }
+ CHILD_PROCESSES.clear();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/631f1dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 67903f7..88859b5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.util;
+import com.google.common.base.Supplier;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
import org.junit.Assert;
@@ -471,4 +472,55 @@ public class TestShell extends Assert {
assertEquals("'foo'\\''bar'", Shell.bashQuote("foo'bar"));
assertEquals("''\\''foo'\\''bar'\\'''", Shell.bashQuote("'foo'bar'"));
}
+
+ @Test(timeout=120000)
+ public void testShellKillAllProcesses() throws Throwable {
+ Assume.assumeFalse(WINDOWS);
+ StringBuffer sleepCommand = new StringBuffer();
+ sleepCommand.append("sleep 200");
+ String[] shellCmd = {"bash", "-c", sleepCommand.toString()};
+ final ShellCommandExecutor shexc1 = new ShellCommandExecutor(shellCmd);
+ final ShellCommandExecutor shexc2 = new ShellCommandExecutor(shellCmd);
+
+ Thread shellThread1 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ shexc1.execute();
+ } catch(IOException ioe) {
+ //ignore IOException from thread interrupt
+ }
+ }
+ };
+ Thread shellThread2 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ shexc2.execute();
+ } catch(IOException ioe) {
+ //ignore IOException from thread interrupt
+ }
+ }
+ };
+
+ shellThread1.start();
+ shellThread2.start();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return shexc1.getProcess() != null;
+ }
+ }, 10, 10000);
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return shexc2.getProcess() != null;
+ }
+ }, 10, 10000);
+
+ Shell.destroyAllProcesses();
+ shexc1.getProcess().waitFor();
+ shexc2.getProcess().waitFor();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/14] hadoop git commit: YARN-5646. Add documentation and update
config parameter names for scheduling of OPPORTUNISTIC containers.
(Konstantinos Karanasos via asuresh)
Posted by xg...@apache.org.
YARN-5646. Add documentation and update config parameter names for scheduling of OPPORTUNISTIC containers. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2273a74c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2273a74c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2273a74c
Branch: refs/heads/YARN-5734
Commit: 2273a74c1f3895163046cca09ff5e983df301d22
Parents: cee0c46
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 16 08:14:34 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Dec 16 08:14:34 2016 -0800
----------------------------------------------------------------------
.../v2/app/rm/RMContainerAllocator.java | 4 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +-
.../hadoop/mapred/TestMROpportunisticMaps.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 59 ++---
.../api/impl/TestDistributedScheduling.java | 1 -
.../src/main/resources/yarn-default.xml | 28 +--
.../yarn/server/nodemanager/NodeManager.java | 2 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 4 +-
.../scheduler/ContainerScheduler.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 16 +-
.../yarn/server/resourcemanager/MockRM.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 8 +-
.../site/markdown/OpportunisticContainers.md | 225 +++++++++++++++++++
14 files changed, 296 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 200fa1f..2b63b9e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -238,8 +238,8 @@ public class RMContainerAllocator extends RMContainerRequestor
// first attempt to contact RM.
retrystartTime = System.currentTimeMillis();
this.scheduledRequests.setNumOpportunisticMapsPer100(
- conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PER_100,
- MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100));
+ conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE,
+ MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE));
LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
"% of the mappers will be scheduled using OPPORTUNISTIC containers");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 2ad86ef..32bbe96 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1005,9 +1005,9 @@ public interface MRJobConfig {
* requested by the AM will be opportunistic. If the total number of maps
* for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
*/
- public static final String MR_NUM_OPPORTUNISTIC_MAPS_PER_100 =
- "mapreduce.job.num-opportunistic-maps-per-100";
- public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100 = 0;
+ public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE =
+ "mapreduce.job.num-opportunistic-maps-percentage";
+ public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0;
/**
* A comma-separated list of properties whose value will be redacted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
index d975fd0..462ff04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
@@ -145,7 +145,7 @@ public class TestMROpportunisticMaps {
job.setNumReduceTasks(numReducers);
// All OPPORTUNISTIC
- job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PER_100, percent);
+ job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent);
job.setInt("mapreduce.map.maxattempts", 1);
job.setInt("mapreduce.reduce.maxattempts", 1);
job.setInt("mapred.test.num_lines", numLines);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index dc7c629..5adbfa7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -312,69 +312,65 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
- /** Setting that controls whether distributed scheduling is enabled or not. */
- public static final String DIST_SCHEDULING_ENABLED =
- YARN_PREFIX + "distributed-scheduling.enabled";
- public static final boolean DIST_SCHEDULING_ENABLED_DEFAULT = false;
-
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED =
- YARN_PREFIX + "opportunistic-container-allocation.enabled";
+ RM_PREFIX + "opportunistic-container-allocation.enabled";
public static final boolean
- OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false;
+ DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED = false;
/** Number of nodes to be used by the Opportunistic Container allocator for
* dispatching containers during container allocation. */
public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
- YARN_PREFIX + "opportunistic-container-allocation.nodes-used";
- public static final int OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED_DEFAULT =
+ RM_PREFIX + "opportunistic-container-allocation.nodes-used";
+ public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
10;
/** Frequency for computing least loaded NMs. */
public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS =
- YARN_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
+ RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
public static final long
- NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS_DEFAULT = 1000;
+ DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS = 1000;
- /** Comparator for determining node load for Distributed Scheduling. */
+ /** Comparator for determining node load for scheduling of opportunistic
+ * containers. */
public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
- YARN_PREFIX + "nm-container-queuing.load-comparator";
- public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR_DEFAULT =
+ RM_PREFIX + "nm-container-queuing.load-comparator";
+ public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
"QUEUE_LENGTH";
/** Value of standard deviation used for calculation of queue limit
* thresholds. */
public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV =
- YARN_PREFIX + "nm-container-queuing.queue-limit-stdev";
- public static final float NM_CONTAINER_QUEUING_LIMIT_STDEV_DEFAULT =
+ RM_PREFIX + "nm-container-queuing.queue-limit-stdev";
+ public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV =
1.0f;
/** Min length of container queue at NodeManager. This is a cluster-wide
* configuration that acts as the lower-bound of optimal queue length
* calculated by the NodeQueueLoadMonitor */
public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH =
- YARN_PREFIX + "nm-container-queuing.min-queue-length";
- public static final int NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH_DEFAULT = 1;
+ RM_PREFIX + "nm-container-queuing.min-queue-length";
+ public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5;
/** Max length of container queue at NodeManager. This is a cluster-wide
* configuration that acts as the upper-bound of optimal queue length
* calculated by the NodeQueueLoadMonitor */
public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH =
- YARN_PREFIX + "nm-container-queuing.max-queue-length";
- public static final int NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH_DEFAULT = 10;
+ RM_PREFIX + "nm-container-queuing.max-queue-length";
+ public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15;
/** Min queue wait time for a container at a NodeManager. */
public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
- YARN_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
- public static final int NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS_DEFAULT =
- 1;
+ RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
+ public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
+ 10;
/** Max queue wait time for a container queue at a NodeManager. */
public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
- YARN_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
- public static final int NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS_DEFAULT =
- 10;
+ RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
+ public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
+ 100;
/**
* Enable/disable intermediate-data encryption at YARN level. For now, this
@@ -812,9 +808,14 @@ public class YarnConfiguration extends Configuration {
/** Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM. */
public static final String NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
NM_PREFIX + "opportunistic-containers-max-queue-length";
- public static final int NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH_DEFAULT =
+ public static final int DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
0;
+ /** Setting that controls whether distributed scheduling is enabled or not. */
+ public static final String DIST_SCHEDULING_ENABLED =
+ NM_PREFIX + "distributed-scheduling.enabled";
+ public static final boolean DEFAULT_DIST_SCHEDULING_ENABLED = false;
+
/** Environment variables that will be sent to containers.*/
public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env";
public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX";
@@ -2844,14 +2845,14 @@ public class YarnConfiguration extends Configuration {
public static boolean isDistSchedulingEnabled(Configuration conf) {
return conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
- YarnConfiguration.DIST_SCHEDULING_ENABLED_DEFAULT);
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
}
public static boolean isOpportunisticContainerAllocationEnabled(
Configuration conf) {
return conf.getBoolean(
YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED,
- YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT);
+ YarnConfiguration.DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED);
}
// helper methods for timeline service configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
index b552d19..e180f6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
@@ -104,7 +104,6 @@ public class TestDistributedScheduling extends BaseAMRMProxyE2ETest {
cluster = new MiniYARNCluster("testDistributedSchedulingE2E", 1, 1, 1);
conf = new YarnConfiguration();
- conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
conf.setBoolean(YarnConfiguration.
OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index c8c4edd..3e739ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2751,7 +2751,7 @@
<description>
Setting that controls whether distributed scheduling is enabled.
</description>
- <name>yarn.distributed-scheduling.enabled</name>
+ <name>yarn.nodemanager.distributed-scheduling.enabled</name>
<value>false</value>
</property>
@@ -2760,7 +2760,7 @@
Setting that controls whether opportunistic container allocation
is enabled.
</description>
- <name>yarn.opportunistic-container-allocation.enabled</name>
+ <name>yarn.resourcemanager.opportunistic-container-allocation.enabled</name>
<value>false</value>
</property>
@@ -2769,7 +2769,7 @@
Number of nodes to be used by the Opportunistic Container Allocator for
dispatching containers during container allocation.
</description>
- <name>yarn.opportunistic-container-allocation.nodes-used</name>
+ <name>yarn.resourcemanager.opportunistic-container-allocation.nodes-used</name>
<value>10</value>
</property>
@@ -2777,7 +2777,7 @@
<description>
Frequency for computing least loaded NMs.
</description>
- <name>yarn.nm-container-queuing.sorting-nodes-interval-ms</name>
+ <name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
<value>1000</value>
</property>
@@ -2785,7 +2785,7 @@
<description>
Comparator for determining node load for Distributed Scheduling.
</description>
- <name>yarn.nm-container-queuing.load-comparator</name>
+ <name>yarn.resourcemanager.nm-container-queuing.load-comparator</name>
<value>QUEUE_LENGTH</value>
</property>
@@ -2793,7 +2793,7 @@
<description>
Value of standard deviation used for calculation of queue limit thresholds.
</description>
- <name>yarn.nm-container-queuing.queue-limit-stdev</name>
+ <name>yarn.resourcemanager.nm-container-queuing.queue-limit-stdev</name>
<value>1.0f</value>
</property>
@@ -2801,32 +2801,32 @@
<description>
Min length of container queue at NodeManager.
</description>
- <name>yarn.nm-container-queuing.min-queue-length</name>
- <value>1</value>
+ <name>yarn.resourcemanager.nm-container-queuing.min-queue-length</name>
+ <value>5</value>
</property>
<property>
<description>
Max length of container queue at NodeManager.
</description>
- <name>yarn.nm-container-queuing.max-queue-length</name>
- <value>10</value>
+ <name>yarn.resourcemanager.nm-container-queuing.max-queue-length</name>
+ <value>15</value>
</property>
<property>
<description>
Min queue wait time for a container at a NodeManager.
</description>
- <name>yarn.nm-container-queuing.min-queue-wait-time-ms</name>
- <value>1</value>
+ <name>yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms</name>
+ <value>10</value>
</property>
<property>
<description>
Max queue wait time for a container queue at a NodeManager.
</description>
- <name>yarn.nm-container-queuing.max-queue-wait-time-ms</name>
- <value>10</value>
+ <name>yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms</name>
+ <value>100</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 72875a9..7de5317 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -329,7 +329,7 @@ public class NodeManager extends CompositeService
boolean isDistSchedulingEnabled =
conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
- YarnConfiguration.DIST_SCHEDULING_ENABLED_DEFAULT);
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
this.context = createNMContext(containerTokenSecretManager,
nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 79882aa..dc56090 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -79,7 +79,7 @@ import com.google.common.base.Preconditions;
* to intercept and inspect messages from application master to the cluster
* resource manager. It listens to messages from the application master and
* creates a request intercepting pipeline instance for each application. The
- * pipeline is a chain of intercepter instances that can inspect and modify the
+ * pipeline is a chain of interceptor instances that can inspect and modify the
* request/response as needed.
*/
public class AMRMProxyService extends AbstractService implements
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index e4d9346..8f4478e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -305,7 +305,9 @@ public class ContainerManagerImpl extends CompositeService implements
protected void createAMRMProxyService(Configuration conf) {
this.amrmProxyEnabled =
conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,
- YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED);
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED) ||
+ conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
if (amrmProxyEnabled) {
LOG.info("AMRMProxyService is enabled. "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 753bf04..24530b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -106,7 +106,7 @@ public class ContainerScheduler extends AbstractService implements
this(context, dispatcher, metrics, context.getConf().getInt(
YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
YarnConfiguration.
- NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH_DEFAULT));
+ DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 7b7963f..a527d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -112,11 +112,11 @@ public class OpportunisticContainerAllocatorAMService
rmContext.getContainerTokenSecretManager());
this.k = rmContext.getYarnConfiguration().getInt(
YarnConfiguration.OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED,
- YarnConfiguration.OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED_DEFAULT);
+ YarnConfiguration.DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED);
long nodeSortInterval = rmContext.getYarnConfiguration().getLong(
YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS,
YarnConfiguration.
- NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS_DEFAULT);
+ DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS);
this.cacheRefreshInterval = nodeSortInterval;
this.lastCacheUpdateTime = System.currentTimeMillis();
NodeQueueLoadMonitor.LoadComparator comparator =
@@ -124,14 +124,14 @@ public class OpportunisticContainerAllocatorAMService
rmContext.getYarnConfiguration().get(
YarnConfiguration.NM_CONTAINER_QUEUING_LOAD_COMPARATOR,
YarnConfiguration.
- NM_CONTAINER_QUEUING_LOAD_COMPARATOR_DEFAULT));
+ DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR));
NodeQueueLoadMonitor topKSelector =
new NodeQueueLoadMonitor(nodeSortInterval, comparator);
float sigma = rmContext.getYarnConfiguration()
.getFloat(YarnConfiguration.NM_CONTAINER_QUEUING_LIMIT_STDEV,
- YarnConfiguration.NM_CONTAINER_QUEUING_LIMIT_STDEV_DEFAULT);
+ YarnConfiguration.DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV);
int limitMin, limitMax;
@@ -139,22 +139,22 @@ public class OpportunisticContainerAllocatorAMService
limitMin = rmContext.getYarnConfiguration()
.getInt(YarnConfiguration.NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH,
YarnConfiguration.
- NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH_DEFAULT);
+ DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH);
limitMax = rmContext.getYarnConfiguration()
.getInt(YarnConfiguration.NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH,
YarnConfiguration.
- NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH_DEFAULT);
+ DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH);
} else {
limitMin = rmContext.getYarnConfiguration()
.getInt(
YarnConfiguration.NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS,
YarnConfiguration.
- NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS_DEFAULT);
+ DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS);
limitMax = rmContext.getYarnConfiguration()
.getInt(
YarnConfiguration.NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS,
YarnConfiguration.
- NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS_DEFAULT);
+ DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS);
}
topKSelector.initThresholdCalculator(sigma, limitMin, limitMax);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index a66b093..02d3956 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -947,7 +947,7 @@ public class MockRM extends ResourceManager {
protected ApplicationMasterService createApplicationMasterService() {
if (this.rmContext.getYarnConfiguration().getBoolean(
YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED,
- YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT)) {
+ YarnConfiguration.DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED)) {
return new OpportunisticContainerAllocatorAMService(getRMContext(),
scheduler) {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 4f484ad..5ae74a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -852,7 +852,9 @@ public class MiniYARNCluster extends CompositeService {
protected void createAMRMProxyService(Configuration conf) {
this.amrmProxyEnabled =
conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,
- YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED);
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED) ||
+ conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
if (this.amrmProxyEnabled) {
LOG.info("CustomAMRMProxyService is enabled. "
@@ -882,7 +884,9 @@ public class MiniYARNCluster extends CompositeService {
protected void createAMRMProxyService(Configuration conf) {
this.amrmProxyEnabled =
conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,
- YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED);
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED) ||
+ conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
if (this.amrmProxyEnabled) {
LOG.info("CustomAMRMProxyService is enabled. "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2273a74c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
new file mode 100644
index 0000000..ac26d88
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
@@ -0,0 +1,225 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Opportunistic Containers
+========================
+
+* [Purpose](#Purpose)
+* [Quick Guide](#Quick_Guide)
+ * [Main Goal](#Main_Goal)
+ * [Enabling Opportunistic Containers](#Enabling_Opportunistic_Containers)
+ * [Running a Sample Job](Running_a_Sample_Job)
+ * [Opportunistic Containers in Web UI](Opportunistic_Containers_in_Web_UI)
+* [Overview](#Overview)
+* [Container Execution Types](#Container_Execution_Types)
+* [Execution of Opportunistic Containers](#Execution_of_Opportunistic_Containers)
+* [Allocation of Opportunistic Containers](#Allocation_of_Opportunistic_Containers)
+ * [Centralized Allocation](#Centralized_Allocation)
+ * [Distributed Allocation](#Distributed_Allocation)
+ * [Determining Nodes for Allocation](#Determining_Nodes_for_Allocation)
+ * [Rebalancing Node Load](#Rebalancing_Node_Load)
+* [Advanced Configuration](#Advanced_Configuration)
+* [Items for Future Work](#Items_for_Future_Work)
+
+
+<a name="Purpose"></a>Purpose
+-----------------------------
+
+This document introduces the notion of **opportunistic** container execution, and discusses how opportunistic containers are allocated and executed.
+
+
+<a name="Quick_Guide"></a>Quick Guide
+--------------------------------------------------------------------
+
+We start by providing a brief overview of opportunistic containers, including how a user can enable this feature and run a sample job using such containers.
+
+###<a name="Main_Goal"></a>Main Goal
+
+Unlike existing YARN containers that are scheduled in a node only if there are unallocated resources, opportunistic containers can be dispatched to an NM, even if their execution at that node cannot start immediately. In such a case, opportunistic containers will be queued at that NM until resources become available.
+The main goal of opportunistic container execution is to improve cluster resource utilization, and therefore increase task throughput. Resource utilization and task throughput improvements are more pronounced for workloads that include relatively short tasks (in the order of seconds).
+
+
+###<a name="Enabling_Opportunistic_Containers"></a>Enabling Opportunistic Containers
+
+To enable opportunistic container allocation, the following two properties have to be present in **conf/yarn-site.xml**:
+
+| Property | Description | Default value |
+|:-------- |:----- |:----- |
+| `yarn.resourcemanager.opportunistic-container-allocation.enabled` | Enables opportunistic container allocation. | `false` |
+| `yarn.nodemanager.opportunistic-containers-max-queue-length` | Determines the max number of opportunistic containers that can be queued at an NM. | `0` |
+
+The first parameter above has to be set to `true`. The second one has to be set to a positive value to allow queuing of opportunistic containers at the NM. A value of `10` can be used to start experimenting with opportunistic containers. The optimal value depends on the jobs characteristics, the cluster configuration and the target utilization.
+
+By default, allocation of opportunistic containers is performed centrally through the RM. However, a user can choose to enable distributed allocation of opportunistic containers, which can further improve allocation latency for short tasks. Distributed scheduling can be enabling by setting to `true` the following parameter (note that non-opportunistic containers will continue being scheduled through the RM):
+
+| Property | Description | Default value |
+|:-------- |:----- |:----- |
+| `yarn.nodemanager.distributed-scheduling.enabled` | Enables distributed scheduling. | `false` |
+
+
+###<a name="Running_a_Sample_Job"></a>Running a Sample Job
+
+The following command can be used to run a sample pi map-reduce job, executing 40% of mappers using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version of Hadoop you are using):
+```
+$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100
+```
+
+By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers.
+
+
+###<a name="Opportunistic_Containers_in_Web_UI"></a>Opportunistic Containers in Web UI
+
+When opportunistic container allocation is enabled, the following new columns can be observed in the Nodes page of the Web UI (`rm-address:8088/cluster/nodes`):
+* Running Containers (O): number of running opportunistic containers on each node;
+* Mem Used (O): Total memory used by opportunistic containers on each node;
+* VCores Used (O): Total CPU virtual cores used by opportunistic containers on each node;
+* Queued Containers: Number of containers queued at each node.
+
+When clicking on a specific container running on a node, the execution type of the container is also shown.
+
+In the rest of the document, we provide an in-depth description of opportunistic containers, including details about their allocation and execution.
+
+
+Overview <a name="Overview"></a>
+--------------------------------
+
+The existing schedulers in YARN (Fair and Capacity Scheduler) allocate containers to a node only if there are unallocated resources at that node at the moment of scheduling the containers. This **guaranteed** type of execution has the advantage that once the AM dispatches a container to a node, the container execution will start immediately, since it is guaranteed that there will be available resources. Moreover, unless fairness or capacity constraints are violated, containers are guaranteed to run to completion without being preempted.
+
+Although this design offers a more predictable task execution, it has two main drawbacks that can lead to suboptimal cluster resource utilization:
+
+* **Feedback delays.** When a container finishes its execution at a node, the RM gets notified that there are available resources through the next NM-RM heartbeat, then the RM schedules a new container at that node, the AM gets notified through the next AM-RM heartbeat, and finally the AM launches the new container at the node. These delays result in idle node resources, which in turn lead to lower resource utilization, especially when workloads involve tasks whose duration is relatively short.
+* **Allocated vs. utilized resources.** The RM allocates containers based on the *allocated* resources at each node, which might be significantly higher than the actually *utilized* resources (e.g., think of a container for which 4GB memory have been allocated, but only 2GB are being utilized). This lowers effective resource utilization, and can be avoided if the RM takes into account the utilized resources during scheduling. However, this has to be done in a way that allows resources to be reclaimed in case the utilized resources of a running container increase.
+
+To mitigate the above problems, in addition to the existing containers (which we term **guaranteed** containers hereafter), we introduce the notion of **opportunistic** containers. An opportunistic container can be dispatched to an NM, even if there are no available (unallocated) resources for it at the moment of scheduling. In such a case, the opportunistic container will be queued at the NM, waiting for resources to become available for its execution to start. The opportunistic containers are of lower priority than the guaranteed ones, which means that they can be preempted for guaranteed containers to start their execution. Therefore, they can be used to improve cluster resource utilization without impacting the execution of existing guaranteed containers.
+
+An additional advantage of opportunistic containers is that they introduce a notion of **execution priority at the NMs**. For instance, a lower priority job that does not require strict execution guarantees can use opportunistic containers or a mix of container execution types for its tasks.
+
+We have introduced two ways of allocating opportunistic containers: a **centralized** and a **distributed** one. In the centralized scheduling, opportunistic containers are allocated through the YARN RM, whereas in the distributed one, through local schedulers that reside at each NM. Centralized allocation allows for higher quality placement decisions and for implementing more involved sharing policies across applications (e.g., fairness). On the other hand, distributed scheduling can offer faster container allocation, which is useful for short tasks, as it avoids the round-trip to the RM. In both cases, the scheduling of guaranteed containers remains intact and happens through the YARN RM (using the existing Fair or Capacity Scheduler).
+
+Note that in the current implementation, we are allocating containers based on allocated (and not utilized) resources. Therefore, we tackle the "feedback delays" problem mentioned above, but not the "allocated vs. utilized resources" one. There is ongoing work (`YARN-1011`) that employs opportunistic containers to address the latter problem too.
+
+Below, we describe in more detail the [container execution types](#Container_Execution_Types), as well as the [execution](#Execution_of_Opportunistic_Containers) (including the container queuing at the NMs) and [allocation](#Allocation_of_Opportunistic_Containers) of opportunistic containers. Then we discuss how to fine-tune opportunistic containers through some [advanced configuration parameters](#Advanced_Configuration). Finally, we discuss open items for [future work](#Items_for_Future_Work).
+
+
+<a name="Container_Execution_Types"></a>Container Execution Types
+-----------------------------------------------------------------
+
+We introduce the following two types of containers:
+
+* **Guaranteed containers** correspond to the existing YARN containers. They are allocated by the Fair or Capacity Scheduler, and once dispatched to a node, it is guaranteed that there are available resources for their execution to start immediately. Moreover, these containers run to completion (as long as there are no failures). They can be preempted only in case the scheduler's queue to which they belong, violates fairness or capacity constraints.
+* **Opportunistic containers** are not guaranteed to have resources for their execution to start when they get dispatched to a node. Instead, they might be queued at the NM until resources become available. In case a guaranteed container arrives at a node and there are no resources available for it, one or more opportunistic containers will be preempted to execute the guaranteed one.
+
+When an AM submits its resource requests to the RM, it specifies the type for each container (default is guaranteed), determining the way the container will be [allocated](#Allocation_of_Opportunistic_Containers). Subsequently, when the container is launched by the AM at an NM, its type determines how it will be [executed](#Execution_of_Opportunistic_Containers) by the NM.
+
+
+<a name="Execution_of_Opportunistic_Containers"></a>Execution of Opportunistic Containers
+---------------------------------------------------------------------------
+
+When a container arrives at an NM, its execution is determined by the available resources at the NM and the container type. Guaranteed containers start their execution immediately, and if needed, the NM will kill running opportunistic containers to ensure there are sufficient resources for the guaranteed ones to start. On the other hand, opportunistic containers can be queued at the NM, if there are no resources available to start their execution when they arrive at the NM. To enable this, we extended the NM by allowing queuing of containers at each node. The NM monitors the local resources, and when there are sufficient resources available, it starts the execution of the opportunistic container that is at the head of the queue.
+
+In particular, when a container arrives at an NM, localization is performed (i.e., all required resources are downloaded), and then the container moves to a `SCHEDULED` state, in which the container is queued, waiting for its execution to begin:
+
+* If there are available resources, the execution of the container starts immediately, irrespective of its execution type.
+* If there are no available resources:
+ * If the container is guaranteed, we kill as many running opportunistic containers as required for the guaranteed container to be executed, and then start its execution.
+ * If the container is opportunistic, it remains at the queue until resources become available.
+* When a container (guaranteed or opportunistic) finishes its execution and resources get freed up, we examine the queued containers and if there are available resources we start their execution. We pick containers from the queue in a FIFO order.
+
+In the [future work items](#Items_for_Future_Work) below, we discuss different ways of prioritizing task execution (queue reordering) and of killing opportunistic containers to make space for guaranteed ones.
+
+
+<a name="Allocation_of_Opportunistic_Containers"></a>Allocation of Opportunistic Containers
+-----------------------------------------------------------------------------
+
+As mentioned above, we provide both a centralized and a distributed way of allocating opportunistic containers, which we describe below.
+
+###<a name="Centralized_Allocation"></a>Centralized Allocation
+
+We have introduced a new service at the RM, namely the `OpportunisticContainerAllocatorAMService`, which extends the `ApplicationMasterService`. When the centralized opportunistic allocation is enabled, the resource requests from the AMs are served at the RM side by the `OpportunisticContainerAllocatorAMService`, which splits them into two sets of resource requests:
+
+* The guaranteed set is forwarded to the existing `ApplicationMasterService` and is subsequently handled by the Fair or Capacity Scheduler.
+* The opportunistic set is handled by the new `OpportunisticContainerAllocator`, which performs the scheduling of opportunistic containers to nodes.
+
+The `OpportunisticContainerAllocator` maintains a list with the [least loaded nodes](#Determining_Nodes_for_Allocation) of the cluster at each moment, and assigns containers to them in a round-robin fashion. Note that in the current implementation, we purposely do not take into account node locality constraints. Since an opportunistic container (unlike the guaranteed ones) might wait at the queue of an NM before its execution starts, it is more important to allocate it at a node that is less loaded (i.e., where queuing delay will be smaller) rather than respect its locality constraints. Moreover, we do not take into account sharing (fairness/capacity) constraints for opportunistic containers at the moment. Support for both locality and sharing constraints can be added in the future if required.
+
+
+###<a name="Distributed_Allocation"></a>Distributed Allocation
+
+In order to enable distributed scheduling of opportunistic containers, we have introduced a new service at each NM, called `AMRMProxyService`. The `AMRMProxyService` implements the `ApplicationMasterService` protocol, and acts as a proxy between the AMs running at that node and the RM. When the `AMRMProxyService` is enabled (through a parameter), we force all AMs running at a particular node to communicate with the `AMRMProxyService` of the same node, instead of going directly to the RM. Moreover, to ensure that the AMs will not talk directly with the RM, when a new AM gets initialized, we replace its `AMRMToken` with a token signed by the `AMRMProxyService`.
+
+A chain of interceptors can be registered with the `AMRMProxyService`. One of these interceptors is the `DistributedScheduler` that is responsible for allocating opportunistic containers in a distributed way, without needing to contact the RM. This modular design makes the `AMRMProxyService` instrumental in other scenarios too, such as YARN federation (`YARN-2915`) or throttling down misbehaving AMs, which can be enabled simply by adding additional interceptors at the interceptor chain.
+
+When distributed opportunistic scheduling is enabled, each AM sends its resource requests to the `AMRMProxyService` running at the same node. The `AMRMProxyService` splits the resource requests into two sets:
+
+* The guaranteed set is forwarded to the RM. In this case the `AMRMProxyService` simply acts as a proxy between the AM and the RM, and the container allocation remains intact (using the Fair or Capacity Scheduler).
+* The opportunistic set is not forwarded to the RM. Instead, it is handled by the `DistributedScheduler` that is running locally at the node. In particular, the `DistributedScheduler` maintains a list with the least loaded nodes in the cluster, and allocates containers to them in a round-robin fashion. The RM informs the `DistributedScheduler` about the least loaded nodes at regular intervals through the NM-RM heartbeats.
+
+The above procedure is similar to the one performed by the `OpportunisticContainerAllocatorAMService` in the case of centralized opportunistic scheduling described above. The main difference is that in the distributed case, the splitting of requests into guaranteed and opportunistic happens locally at the node, and only the guaranteed requests are forwarded to the RM, while the opportunistic ones are handled without contacting the RM.
+
+
+###<a name="Determining_Nodes_for_Allocation"></a>Determining Nodes for Allocation
+
+Each NM informs the RM periodically through the NM-RM heartbeats about the number of running guaranteed and opportunistic containers, as well as the number of queued opportunistic containers. The RM gathers this information from all nodes and determines the least loaded ones.
+
+In the case of centralized allocation of opportunistic containers, this information is immediately available, since the allocation happens centrally. In the case of distributed scheduling, the list with the least loaded nodes is propagated to all NMs (and thus becomes available to the `DistributedSchedulers`) through the heartbeat responses from the RM to the NMs. The number of least loaded nodes sent to the NMs is configurable.
+
+At the moment, we take into account only the number of queued opportunistic containers at each node in order to estimate the time an opportunistic container would have to wait if sent to that node and, thus, determine the least loaded nodes. If the AM provided us with information about the estimated task durations, we could take them into account in order to have better estimates of the queue waiting times.
+
+
+###<a name="Rebalancing_Node_Load"></a>Rebalancing Node Load
+
+Occasionally poor placement choices for opportunistic containers may be made (due to stale queue length estimates), which can lead to load imbalance between nodes. The problem is more pronounced under high cluster load, and also in the case of distributed scheduling (multiple `DistributedSchedulers` may place containers at the same NM, since they do not coordinate with each other). To deal with this load imbalance between the NM queues, we perform load shedding to dynamically re-balance the load between NMs. In particular, while aggregating at the RM the queue time estimates published by each NM, we construct a distribution and find a targeted maximal value for the length of the NM queues (based on the mean and standard deviation of the distribution). Then the RM disseminates this value to the various NMs through the heartbeat responses. Subsequently, using this information, an NM on a node whose queue length is above the threshold discards opportunistic containers to meet this maxi
mal value. This forces the associated individual AMs to reschedule those containers elsewhere.
+
+
+<a name="Advanced_Configuration"></a>Advanced Configuration
+--------------------------------------------------
+
+The main properties for enabling opportunistic container allocation and choosing between centralized and distributed allocation were described in the [quick guide](#Quick_Guide) in the beginning of this document. Here we present more advanced configuration. Note that using default values for those parameters should be sufficient in most cases. All parameters below have to be defined in the **conf/yarn-site.xml** file.
+
+To determine the number of [least loaded nodes](#Determining_Nodes_for_Allocation) that will be used when scheduling opportunistic containers and how often this list will be refreshed, we use the following parameters:
+
+| Property | Description | Default value |
+|:-------- |:----- |:----- |
+| `yarn.resourcemanager.opportunistic-container-allocation.nodes-used` | Number of least loaded nodes to be used by the Opportunistic Container allocator for dispatching containers during container allocation. A higher value can improve load balance in large clusters. | `10` |
+| `yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms` | Frequency for computing least loaded nodes. | `1000` |
+
+
+As discussed in the [node load rebalancing](#Rebalancing_Node_Load) section above, at regular intervals, the RM gathers all NM queue lengths and computes their mean value (`avg`) and standard deviation (`stdev`), as well as the value `avg + k*stdev` (where `k` a float). This value gets propagated through the NM-RM heartbeats to all NMs, who should respect that value by dequeuing containers (if required), as long as their current queue length is between a `queue_min_length` and a `queue_max_length` value (these values are used to avoid dequeuing tasks from very short queues and to aggressively dequeue tasks from long queues, respectively).
+The parameters `k`, `queue_min_length` and `queue_max_length` can be specified as follows:
+
+| Property | Description | Default value |
+|:-------- |:----- |:----- |
+| `yarn.resourcemanager.nm-container-queuing.queue-limit-stdev` | The `k` parameter. | `1.0f` |
+| `yarn.resourcemanager.nm-container-queuing.min-queue-length` | The `queue_min_length` parameter. | `5` |
+| `yarn.resourcemanager.nm-container-queuing.max-queue-length` | The `queue_max_length` parameter. | `15` |
+
+
+Finally, two more properties can further tune the `AMRMProxyService` in case distributed scheduling is used:
+
+| Property | Description | Default value |
+|:-------- |:----- |:----- |
+| `yarn.nodemanager.amrmproxy.address` | The address/port to which the `AMRMProxyService` is bound to. | `0.0.0.0:8049` |
+| `yarn.nodemanager.amrmproxy.client.thread-count` | The number of threads that are used at each NM for serving the interceptors register to the `AMRMProxyService` by different jobs. | `3` |
+
+
+<a name="Items_for_Future_Work"></a>Items for Future Work
+-----------------------------------------------
+
+Here we describe multiple ways in which we can extend/enhance the allocation and execution of opportunistic containers. We also provide the JIRAs that track each item.
+
+* **Resource overcommitment** (`YARN-1011`). As already discussed, in order to further improve the cluster resource utilization, we can schedule containers not based on the allocated resources but on the actually utilized ones. When over-committing resources, there is the risk of running out of resources in case we have an increase in the utilized resources of the already running containers. Therefore, opportunistic execution should be used for containers whose allocation goes beyond the capacity of a node. This way, we can choose opportunistic containers to kill for reclaiming resources.
+* **NM Queue reordering** (`YARN-5886`). Instead of executing queued containers in a FIFO order, we can employ reordering strategies that dynamically determine which opportunistic container will be executed next. For example, we can prioritize containers that are expected to be short-running or which belong to applications that are close to completion.
+* **Out of order killing at NMs** (`YARN-5887`). As described above, when we need to free up resources for a guaranteed container to start its execution, we kill opportunistic containers in reverse order of arrival (first the most recently started ones). This might not always be the right decision. For example, we might want to minimize the number of containers killed or to refrain from killing containers of jobs that are very close to completion.
+* **Container pausing** (`YARN-5292`): At the moment we kill opportunistic containers to make room for guaranteed in case of resource contention. In busy clusters this can lower the effective cluster utilization: whenever we kill a running opportunistic container, it has to be restarted, and thus we lose work. To this end, we can instead pause running opportunistic containers. Note that this will require support from the container executor (e.g., the container technology used) and from the application.
+* **Container promotion** (`YARN-5085`). There are cases where changing the execution type of a container during its execution can be beneficial. For instance, an application might submit a container as opportunistic, and when its execution starts, it can request its promotion to a guaranteed container to avoid it getting killed.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/14] hadoop git commit: Revert "HADOOP-13709. Ability to clean up
subprocesses spawned by Shell when the process exits. Contributed by Eric
Badger"
Posted by xg...@apache.org.
Revert "HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the process exits. Contributed by Eric Badger"
This reverts commit 9947aeb60c3dd075544866fd6e4dab0ad8b4afa2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/169bfc09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/169bfc09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/169bfc09
Branch: refs/heads/YARN-5734
Commit: 169bfc09037595610eb000fd3a0cb63cc9deca06
Parents: f92913c
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Dec 15 20:30:45 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Dec 15 20:51:24 2016 +0000
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/util/Shell.java | 24 -------
.../java/org/apache/hadoop/util/TestShell.java | 68 --------------------
2 files changed, 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/169bfc09/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 83877b7..5fc9869 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -26,11 +26,9 @@ import java.io.InputStream;
import java.io.InterruptedIOException;
import java.nio.charset.Charset;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
-import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
@@ -50,8 +48,6 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class Shell {
- private static final Map <Process, Object> CHILD_PROCESSES =
- Collections.synchronizedMap(new WeakHashMap<Process, Object>());
public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
/**
@@ -920,7 +916,6 @@ public abstract class Shell {
} else {
process = builder.start();
}
- CHILD_PROCESSES.put(process, null);
if (timeOutInterval > 0) {
timeOutTimer = new Timer("Shell command timeout");
@@ -1017,7 +1012,6 @@ public abstract class Shell {
LOG.warn("Error while closing the error stream", ioe);
}
process.destroy();
- CHILD_PROCESSES.remove(process);
lastTime = Time.monotonicNow();
}
}
@@ -1316,22 +1310,4 @@ public abstract class Shell {
}
}
}
-
- /**
- * Static method to destroy all running <code>Shell</code> processes
- * Iterates through a list of all currently running <code>Shell</code>
- * processes and destroys them one by one. This method is thread safe and
- * is intended to be used in a shutdown hook.
- */
- public static void destroyAllProcesses() {
- synchronized (CHILD_PROCESSES) {
- for (Process key : CHILD_PROCESSES.keySet()) {
- Process process = key;
- if (key != null) {
- process.destroy();
- }
- }
- CHILD_PROCESSES.clear();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/169bfc09/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 5cc011b..67903f7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.util;
-import com.google.common.base.Supplier;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
import org.junit.Assert;
@@ -472,71 +471,4 @@ public class TestShell extends Assert {
assertEquals("'foo'\\''bar'", Shell.bashQuote("foo'bar"));
assertEquals("''\\''foo'\\''bar'\\'''", Shell.bashQuote("'foo'bar'"));
}
-
- @Test(timeout=120000)
- public void testShellKillAllProcesses() throws Throwable {
- Assume.assumeFalse(WINDOWS);
- StringBuffer sleepCommand = new StringBuffer();
- sleepCommand.append("sleep 200");
- String[] shellCmd = {"bash", "-c", sleepCommand.toString()};
- final ShellCommandExecutor shexc1 = new ShellCommandExecutor(shellCmd);
- final ShellCommandExecutor shexc2 = new ShellCommandExecutor(shellCmd);
-
- Thread shellThread1 = new Thread() {
- @Override
- public void run() {
- try {
- shexc1.execute();
- } catch(IOException ioe) {
- //ignore IOException from thread interrupt
- }
- }
- };
- Thread shellThread2 = new Thread() {
- @Override
- public void run() {
- try {
- shexc2.execute();
- } catch(IOException ioe) {
- //ignore IOException from thread interrupt
- }
- }
- };
-
- shellThread1.start();
- shellThread2.start();
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return shexc1.getProcess() != null;
- }
- }, 10, 10000);
-
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return shexc2.getProcess() != null;
- }
- }, 10, 10000);
-
- Shell.destroyAllProcesses();
- final Process process1 = shexc1.getProcess();
- final Process process2 = shexc2.getProcess();
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return !process1.isAlive();
- }
- }, 10, 10000);
-
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return !process2.isAlive();
- }
- }, 10, 10000);
-
- assertFalse("Process 1 was not killed within timeout", process1.isAlive());
- assertFalse("Process 2 was not killed within timeout", process2.isAlive());
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/14] hadoop git commit: HDFS-11160. VolumeScanner reports
write-in-progress replicas as corrupt incorrectly. Contributed by Wei-Chiu
Chuang and Yongjun Zhang.
Posted by xg...@apache.org.
HDFS-11160. VolumeScanner reports write-in-progress replicas as corrupt incorrectly. Contributed by Wei-Chiu Chuang and Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aebb9127
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aebb9127
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aebb9127
Branch: refs/heads/YARN-5734
Commit: aebb9127bae872835d057e1c6a6e6b3c6a8be6cd
Parents: 6fce191
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Dec 15 16:32:50 2016 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Dec 15 16:32:50 2016 -0800
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockScanner.java | 10 +-
.../hdfs/server/datanode/BlockSender.java | 18 +++-
.../hdfs/server/datanode/FinalizedReplica.java | 29 ++++++
.../hdfs/server/datanode/VolumeScanner.java | 9 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 ++
.../datanode/fsdataset/impl/FsVolumeImpl.java | 18 +++-
.../server/datanode/SimulatedFSDataset.java | 6 ++
.../hdfs/server/datanode/TestBlockScanner.java | 100 +++++++++++++++++++
.../server/datanode/TestDirectoryScanner.java | 6 ++
.../datanode/extdataset/ExternalVolumeImpl.java | 7 ++
10 files changed, 203 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 21484fb..8081895 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -64,7 +64,15 @@ public class BlockScanner {
/**
* The scanner configuration.
*/
- private final Conf conf;
+ private Conf conf;
+
+ @VisibleForTesting
+ void setConf(Conf conf) {
+ this.conf = conf;
+ for (Entry<String, VolumeScanner> entry : scanners.entrySet()) {
+ entry.getValue().setConf(conf);
+ }
+ }
/**
* The cached scanner configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 99597dc..203ee35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -240,14 +240,23 @@ class BlockSender implements java.io.Closeable {
Preconditions.checkArgument(sendChecksum,
"If verifying checksum, currently must also send it.");
}
-
+
+ // if there is a append write happening right after the BlockSender
+ // is constructed, the last partial checksum maybe overwritten by the
+ // append, the BlockSender need to use the partial checksum before
+ // the append write.
+ ChunkChecksum chunkChecksum = null;
final long replicaVisibleLength;
try(AutoCloseableLock lock = datanode.data.acquireDatasetLock()) {
replica = getReplica(block, datanode);
replicaVisibleLength = replica.getVisibleLength();
+ if (replica instanceof FinalizedReplica) {
+ // Load last checksum in case the replica is being written
+ // concurrently
+ final FinalizedReplica frep = (FinalizedReplica) replica;
+ chunkChecksum = frep.getLastChecksumAndDataLen();
+ }
}
- // if there is a write in progress
- ChunkChecksum chunkChecksum = null;
if (replica.getState() == ReplicaState.RBW) {
final ReplicaInPipeline rbw = (ReplicaInPipeline) replica;
waitForMinLength(rbw, startOffset + length);
@@ -473,7 +482,7 @@ class BlockSender implements java.io.Closeable {
bytesOnDisk));
}
}
-
+
/**
* Converts an IOExcpetion (not subclasses) to SocketException.
* This is typically done to indicate to upper layers that the error
@@ -547,7 +556,6 @@ class BlockSender implements java.io.Closeable {
if (lastDataPacket && lastChunkChecksum != null) {
int start = checksumOff + checksumDataLen - checksumSize;
byte[] updatedChecksum = lastChunkChecksum.getChecksum();
-
if (updatedChecksum != null) {
System.arraycopy(updatedChecksum, 0, buf, start, checksumSize);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java
index 81a4ab4..e3e0450 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -113,4 +115,31 @@ public class FinalizedReplica extends LocalReplica {
throw new UnsupportedOperationException("Replica of type " + getState() +
" does not support createInfo");
}
+
+ /**
+ * gets the last chunk checksum and the length of the block corresponding
+ * to that checksum.
+ * Note, need to be called with the FsDataset lock acquired. May improve to
+ * lock only the FsVolume in the future.
+ * @throws IOException
+ */
+ public ChunkChecksum getLastChecksumAndDataLen() throws IOException {
+ ChunkChecksum chunkChecksum = null;
+ try {
+ byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum(
+ getBlockFile(), getMetaFile());
+ if (lastChecksum != null) {
+ chunkChecksum =
+ new ChunkChecksum(getVisibleLength(), lastChecksum);
+ }
+ } catch (FileNotFoundException e) {
+ // meta file is lost. Try to continue anyway.
+ DataNode.LOG.warn("meta file " + getMetaFile() +
+ " is missing!");
+ } catch (IOException ioe) {
+ DataNode.LOG.warn("Unable to read checksum from meta file " +
+ getMetaFile(), ioe);
+ }
+ return chunkChecksum;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 1e44fb6..8b29fce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -69,7 +69,12 @@ public class VolumeScanner extends Thread {
/**
* The configuration.
*/
- private final Conf conf;
+ private Conf conf;
+
+ @VisibleForTesting
+ void setConf(Conf conf) {
+ this.conf = conf;
+ }
/**
* The DataNode this VolumEscanner is associated with.
@@ -429,6 +434,7 @@ public class VolumeScanner extends Thread {
if (block == null) {
return -1; // block not found.
}
+ LOG.debug("start scanning block {}", block);
BlockSender blockSender = null;
try {
blockSender = new BlockSender(block, 0, -1,
@@ -610,6 +616,7 @@ public class VolumeScanner extends Thread {
break;
}
if (timeout > 0) {
+ LOG.debug("{}: wait for {} milliseconds", this, timeout);
wait(timeout);
if (stopping) {
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 4947ecf..8aa2fd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -402,6 +402,17 @@ public interface FsVolumeSpi
}
/**
+ * Load last partial chunk checksum from checksum file.
+ * Need to be called with FsDataset lock acquired.
+ * @param blockFile
+ * @param metaFile
+ * @return the last partial checksum
+ * @throws IOException
+ */
+ byte[] loadLastPartialChunkChecksum(File blockFile, File metaFile)
+ throws IOException;
+
+ /**
* Compile a list of {@link ScanInfo} for the blocks in
* the block pool with id {@code bpid}.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 74ee063..e1bc886 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -1119,7 +1119,8 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
- private byte[] loadLastPartialChunkChecksum(
+ @Override
+ public byte[] loadLastPartialChunkChecksum(
File blockFile, File metaFile) throws IOException {
// readHeader closes the temporary FileInputStream.
DataChecksum dcs = BlockMetadataHeader
@@ -1135,13 +1136,22 @@ public class FsVolumeImpl implements FsVolumeSpi {
return null;
}
- int offsetInChecksum = BlockMetadataHeader.getHeaderSize() +
- (int)(onDiskLen / bytesPerChecksum * checksumSize);
+ long offsetInChecksum = BlockMetadataHeader.getHeaderSize() +
+ (onDiskLen / bytesPerChecksum) * checksumSize;
byte[] lastChecksum = new byte[checksumSize];
try (RandomAccessFile raf = fileIoProvider.getRandomAccessFile(
this, metaFile, "r")) {
raf.seek(offsetInChecksum);
- raf.read(lastChecksum, 0, checksumSize);
+ int readBytes = raf.read(lastChecksum, 0, checksumSize);
+ if (readBytes == -1) {
+ throw new IOException("Expected to read " + checksumSize +
+ " bytes from offset " + offsetInChecksum +
+ " but reached end of file.");
+ } else if (readBytes != checksumSize) {
+ throw new IOException("Expected to read " + checksumSize +
+ " bytes from offset " + offsetInChecksum + " but read " +
+ readBytes + " bytes.");
+ }
}
return lastChecksum;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index a0041dd..d3efe48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -557,6 +557,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override
+ public byte[] loadLastPartialChunkChecksum(
+ File blockFile, File metaFile) throws IOException {
+ return null;
+ }
+
+ @Override
public LinkedList<ScanInfo> compileReport(String bpid,
LinkedList<ScanInfo> report, ReportCompiler reportCompiler)
throws InterruptedException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
index 6d35cc5..b627800 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
@@ -36,8 +36,12 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeoutException;
import com.google.common.base.Supplier;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils.MaterializedReplica;
@@ -870,4 +874,100 @@ public class TestBlockScanner {
}
info.sem.release(1);
}
+
+ /**
+ * Test concurrent append and scan.
+ * @throws Exception
+ */
+ @Test(timeout=120000)
+ public void testAppendWhileScanning() throws Exception {
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+ Configuration conf = new Configuration();
+ // throttle the block scanner: 1MB per second
+ conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND, 1048576);
+ // Set a really long scan period.
+ conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
+ conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+ TestScanResultHandler.class.getName());
+ conf.setLong(INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS, 0L);
+ final int numExpectedFiles = 1;
+ final int numExpectedBlocks = 1;
+ final int numNameServices = 1;
+ // the initial file length can not be too small.
+ // Otherwise checksum file stream buffer will be pre-filled and
+ // BlockSender will not see the updated checksum.
+ final int initialFileLength = 2*1024*1024+100;
+ final TestContext ctx = new TestContext(conf, numNameServices);
+ // create one file, with one block.
+ ctx.createFiles(0, numExpectedFiles, initialFileLength);
+ final TestScanResultHandler.Info info =
+ TestScanResultHandler.getInfo(ctx.volumes.get(0));
+ String storageID = ctx.volumes.get(0).getStorageID();
+ synchronized (info) {
+ info.sem = new Semaphore(numExpectedBlocks*2);
+ info.shouldRun = true;
+ info.notify();
+ }
+ // VolumeScanner scans the first block when DN starts.
+ // Due to throttler, this should take approximately 2 seconds.
+ waitForRescan(info, numExpectedBlocks);
+
+ // update throttler to schedule rescan immediately.
+ // this number must be larger than initial file length, otherwise
+ // throttler prevents immediate rescan.
+ conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND,
+ initialFileLength+32*1024);
+ BlockScanner.Conf newConf = new BlockScanner.Conf(conf);
+ ctx.datanode.getBlockScanner().setConf(newConf);
+ // schedule the first block for scanning
+ ExtendedBlock first = ctx.getFileBlock(0, 0);
+ ctx.datanode.getBlockScanner().markSuspectBlock(storageID, first);
+
+ // append the file before VolumeScanner completes scanning the block,
+ // which takes approximately 2 seconds to complete.
+ FileSystem fs = ctx.cluster.getFileSystem();
+ FSDataOutputStream os = fs.append(ctx.getPath(0));
+ long seed = -1;
+ int size = 200;
+ final byte[] bytes = AppendTestUtil.randomBytes(seed, size);
+ os.write(bytes);
+ os.hflush();
+ os.close();
+ fs.close();
+
+ // verify that volume scanner does not find bad blocks after append.
+ waitForRescan(info, numExpectedBlocks);
+
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.INFO);
+ }
+
+ private void waitForRescan(final TestScanResultHandler.Info info,
+ final int numExpectedBlocks)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for the first 1 blocks to be scanned.");
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ synchronized (info) {
+ if (info.blocksScanned >= numExpectedBlocks) {
+ LOG.info("info = {}. blockScanned has now reached 1.", info);
+ return true;
+ } else {
+ LOG.info("info = {}. Waiting for blockScanned to reach 1.", info);
+ return false;
+ }
+ }
+ }
+ }, 1000, 30000);
+
+ synchronized (info) {
+ assertEquals("Expected 1 good block.",
+ numExpectedBlocks, info.goodBlocks.size());
+ info.goodBlocks.clear();
+ assertEquals("Expected 1 blocksScanned",
+ numExpectedBlocks, info.blocksScanned);
+ assertEquals("Did not expect bad blocks.", 0, info.badBlocks.size());
+ info.blocksScanned = 0;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index cc0915d..9b0aa82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -899,6 +899,12 @@ public class TestDirectoryScanner {
}
@Override
+ public byte[] loadLastPartialChunkChecksum(
+ File blockFile, File metaFile) throws IOException {
+ return null;
+ }
+
+ @Override
public LinkedList<ScanInfo> compileReport(String bpid,
LinkedList<ScanInfo> report, ReportCompiler reportCompiler)
throws InterruptedException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aebb9127/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
index e607de5..2d33e20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode.extdataset;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.channels.ClosedChannelException;
@@ -109,6 +110,12 @@ public class ExternalVolumeImpl implements FsVolumeSpi {
}
@Override
+ public byte[] loadLastPartialChunkChecksum(
+ File blockFile, File metaFile) throws IOException {
+ return null;
+ }
+
+ @Override
public LinkedList<ScanInfo> compileReport(String bpid,
LinkedList<ScanInfo> report, ReportCompiler reportCompiler)
throws InterruptedException, IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/14] hadoop git commit: HADOOP-11804. Shaded Hadoop client
artifacts and minicluster. Contributed by Sean Busbey.
Posted by xg...@apache.org.
HADOOP-11804. Shaded Hadoop client artifacts and minicluster. Contributed by Sean Busbey.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70ca1f1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70ca1f1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70ca1f1e
Branch: refs/heads/YARN-5734
Commit: 70ca1f1e3a328b18eb4e27f7d0f328ae403342d5
Parents: 36947f7
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Dec 15 11:44:59 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Dec 15 11:44:59 2016 -0800
----------------------------------------------------------------------
.gitignore | 1 +
dev-support/bin/dist-layout-stitching | 8 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 216 ++++++
.../hadoop-client-check-invariants/pom.xml | 124 +++
.../hadoop-client-check-test-invariants/pom.xml | 132 ++++
.../hadoop-client-integration-tests/pom.xml | 130 ++++
.../apache/hadoop/example/ITUseMiniCluster.java | 113 +++
.../src/test/resources/hdfs-site.xml | 34 +
.../src/test/resources/log4j.properties | 24 +
.../hadoop-client-minicluster/pom.xml | 745 +++++++++++++++++++
.../hadoop-client-runtime/pom.xml | 318 ++++++++
hadoop-client-modules/hadoop-client/pom.xml | 338 +++++++++
hadoop-client-modules/pom.xml | 45 ++
hadoop-client/pom.xml | 319 --------
hadoop-dist/pom.xml | 17 +
hadoop-maven-plugins/pom.xml | 22 +
.../resource/ServicesResourceTransformer.java | 164 ++++
.../plugin/shade/resource/package-info.java | 23 +
hadoop-project/pom.xml | 96 ++-
pom.xml | 4 +-
20 files changed, 2544 insertions(+), 329 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d7e321e..eb98a3f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@
.settings
target
build
+dependency-reduced-pom.xml
# External tool builders
*/.externalToolBuilders
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index 40ccdc7..fdbc83c 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -137,6 +137,12 @@ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERS
run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
+# copy client jars as-is
+run mkdir -p "share/hadoop/client"
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
+
echo
echo "Hadoop dist layout available at: ${BASEDIR}/hadoop-${VERSION}"
-echo
\ No newline at end of file
+echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml
new file mode 100644
index 0000000..9342d53
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+<parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+</parent>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <description>Apache Hadoop Client</description>
+ <name>Apache Hadoop Client API</name>
+
+ <properties>
+ <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+ <!-- We contain no source -->
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <!-- We list this as optional because as a type-pom it won't get included in the shading.
+ Marking it optional means it doesn't count as a transitive dependency of this artifact.
+ -->
+ <optional>true</optional>
+ <exclusions>
+ <!-- these APIs are a part of the SE JDK -->
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xml-apis</groupId>
+ <artifactId>xml-apis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- This comes from our parent pom. If we don't expressly change it here to get included,
+ downstream will get warnings at compile time. -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- We contain no source -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <configuration>
+ <skipSource>true</skipSource>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hadoop:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- We get these package level classes from various yarn api jars -->
+ <filter>
+ <artifact>org.apache.hadoop:hadoop-yarn-common</artifact>
+ <excludes>
+ <exclude>org/apache/hadoop/yarn/factories/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/util/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/factory/providers/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/client/api/impl/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/client/api/package-info.class</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+ <excludes>
+ <exclude>org/apache/hadoop/*</exclude>
+ <exclude>org/apache/hadoop/**/*</exclude>
+ <!-- Our non-shaded htrace and logging libraries -->
+ <exclude>org/apache/htrace/*</exclude>
+ <exclude>org/apache/htrace/**/*</exclude>
+ <exclude>org/slf4j/*</exclude>
+ <exclude>org/slf4j/**/*</exclude>
+ <exclude>org/apache/commons/logging/*</exclude>
+ <exclude>org/apache/commons/logging/**/*</exclude>
+ <exclude>org/apache/log4j/*</exclude>
+ <exclude>org/apache/log4j/**/*</exclude>
+ <exclude>**/pom.xml</exclude>
+ <!-- Not the org/ packages that are a part of the jdk -->
+ <exclude>org/ietf/jgss/*</exclude>
+ <exclude>org/omg/**/*</exclude>
+ <exclude>org/w3c/dom/*</exclude>
+ <exclude>org/w3c/dom/**/*</exclude>
+ <exclude>org/xml/sax/*</exclude>
+ <exclude>org/xml/sax/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>com/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Not the com/ packages that are a part of particular jdk implementations -->
+ <exclude>com/sun/tools/*</exclude>
+ <exclude>com/sun/javadoc/*</exclude>
+ <exclude>com/sun/security/*</exclude>
+ <exclude>com/sun/jndi/*</exclude>
+ <exclude>com/sun/management/*</exclude>
+ <exclude>com/sun/tools/**/*</exclude>
+ <exclude>com/sun/javadoc/**/*</exclude>
+ <exclude>com/sun/security/**/*</exclude>
+ <exclude>com/sun/jndi/**/*</exclude>
+ <exclude>com/sun/management/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>io/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>io/compression/*</exclude>
+ <exclude>io/compression/**/*</exclude>
+ <exclude>io/mapfile/*</exclude>
+ <exclude>io/mapfile/**/*</exclude>
+ <exclude>io/map/index/*</exclude>
+ <exclude>io/seqfile/*</exclude>
+ <exclude>io/seqfile/**/*</exclude>
+ <exclude>io/file/buffer/size</exclude>
+ <exclude>io/skip/checksum/errors</exclude>
+ <exclude>io/sort/*</exclude>
+ <exclude>io/serializations</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/servlet/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>net/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>net/topology/*</exclude>
+ <exclude>net/topology/**/*</exclude>
+ </excludes>
+ </relocation>
+ </relocations>
+ <transformers>
+ <!-- Needed until MSHADE-182 -->
+ <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
new file mode 100644
index 0000000..f28816f
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-client-check-invariants</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <description>Enforces our invariants for the api and runtime client modules.</description>
+ <name>Apache Hadoop Client Packaging Invariants</name>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.4</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>extra-enforcer-rules</artifactId>
+ <version>1.0-beta-3</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <banTransitiveDependencies>
+<!--
+ <message>
+ Our client-facing artifacts are not supposed to have additional dependencies
+ and one or more of them do. The output from the enforcer plugin should give
+ specifics.
+ </message>
+-->
+ <excludes>
+ <!-- annotations is provided, and both artifacts exclude the tools transitive,
+ but enforcer still sees it.
+ -->
+ <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+ <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+ <exclude>commons-logging:commons-logging</exclude>
+ <!-- Leave log4j unshaded so downstream users can configure logging. -->
+ <exclude>log4j:log4j</exclude>
+ </excludes>
+ </banTransitiveDependencies>
+ <banDuplicateClasses>
+ <findAllDuplicates>true</findAllDuplicates>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <ignoreClasses>
+ <ignoreClass>*</ignoreClass>
+ </ignoreClasses>
+ </dependency>
+ </dependencies>
+ </banDuplicateClasses>
+ </rules>
+<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
+<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
+ Manually checking the set of Keys that look like packages we relocate:
+
+ cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'` | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
+
+ Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
+
+ for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
+ clazz=${clazz#*src/main/java/}
+ clazz="${clazz%.java}"
+ javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
+ -constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
+ done
+-->
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
new file mode 100644
index 0000000..23ab2ca
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-client-check-test-invariants</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <description>Enforces our invariants for the testing client modules.</description>
+ <name>Apache Hadoop Client Packaging Invariants for Test</name>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.4</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>extra-enforcer-rules</artifactId>
+ <version>1.0-beta-3</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <banTransitiveDependencies>
+<!--
+ <message>
+ Our client-facing artifacts are not supposed to have additional dependencies
+ and one or more of them do. The output from the enforcer plugin should give
+ specifics.
+ </message>
+-->
+ <excludes>
+ <!-- annotations is provided, and both artifacts exclude the tools transitive,
+ but enforcer still sees it.
+ -->
+ <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+ <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+ <exclude>commons-logging:commons-logging</exclude>
+ <!-- Leave log4j unshaded so downstream users can configure logging. -->
+ <exclude>log4j:log4j</exclude>
+ <!-- Leave JUnit unshaded so downstream can use our test helper classes -->
+ <exclude>junit:junit</exclude>
+ <!-- JUnit brings in hamcrest -->
+ <exclude> org.hamcrest:hamcrest-core</exclude>
+ </excludes>
+ </banTransitiveDependencies>
+ <banDuplicateClasses>
+ <findAllDuplicates>true</findAllDuplicates>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <ignoreClasses>
+ <ignoreClass>*</ignoreClass>
+ </ignoreClasses>
+ </dependency>
+ </dependencies>
+ </banDuplicateClasses>
+ </rules>
+<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
+<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
+ Manually checking the set of Keys that look like packages we relocate:
+
+ cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'` | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
+
+ Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
+
+ for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
+ clazz=${clazz#*src/main/java/}
+ clazz="${clazz%.java}"
+ javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
+ -constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
+ done
+-->
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
new file mode 100644
index 0000000..0df6776
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-client-integration-tests</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+
+ <description>Checks that we can use the generated artifacts</description>
+ <name>Apache Hadoop Client Packaging Integration Tests</name>
+
+ <properties>
+ <failsafe.timeout>400</failsafe.timeout>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- Because our tests rely on our shaded artifacts, we can't compile
+ them until after the package phase has run.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <executions>
+ <!--
+ First, let's make sure the normal test-compile doesn't try to
+ compile our integration tests.
+ -->
+ <execution>
+ <id>default-testCompile</id>
+ <phase>test-compile</phase>
+ <configuration>
+ <testExcludes>
+ <testExclude>**/IT*</testExclude>
+ <testExclude>**/*IT</testExclude>
+ </testExcludes>
+ </configuration>
+ </execution>
+ <!--
+ Finally, let's make a 'just for integration tests'-compile that
+ fires off prior to our integration tests but after the package
+ phase has created our shaded artifacts.
+ -->
+ <execution>
+ <id>compile-integration-tests</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <configuration>
+ <testIncludes>
+ <testInclude>**/IT*</testInclude>
+ <testInclude>**/*IT</testInclude>
+ </testIncludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
new file mode 100644
index 0000000..3f560d0
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.hadoop.example;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+
+/**
+ * Ensure that we can perform operations against the shaded minicluster
+ * given the API and runtime jars by performing some simple smoke tests.
+ */
+public class ITUseMiniCluster {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITUseMiniCluster.class);
+
+ private MiniDFSCluster cluster;
+
+ private static final String TEST_PATH = "/foo/bar/cats/dee";
+ private static final String FILENAME = "test.file";
+
+ private static final String TEXT = "Lorem ipsum dolor sit amet, consectetur "
+ + "adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore "
+ + "magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
+ + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute "
+ + "irure dolor in reprehenderit in voluptate velit esse cillum dolore eu "
+ + "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident,"
+ + " sunt in culpa qui officia deserunt mollit anim id est laborum.";
+
+ @Before
+ public void clusterUp() throws IOException {
+ final Configuration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(3)
+ .build();
+ cluster.waitActive();
+ }
+
+ @After
+ public void clusterDown() {
+ cluster.close();
+ }
+
+ @Test
+ public void useHdfsFileSystem() throws IOException {
+ try (final FileSystem fs = cluster.getFileSystem()) {
+ simpleReadAfterWrite(fs);
+ }
+ }
+
+ public void simpleReadAfterWrite(final FileSystem fs) throws IOException {
+ LOG.info("Testing read-after-write with FS implementation: {}", fs);
+ final Path path = new Path(TEST_PATH, FILENAME);
+ if (!fs.mkdirs(path.getParent())) {
+ throw new IOException("Mkdirs failed to create " +
+ TEST_PATH);
+ }
+ try (final FSDataOutputStream out = fs.create(path)) {
+ out.writeUTF(TEXT);
+ }
+ try (final FSDataInputStream in = fs.open(path)) {
+ final String result = in.readUTF();
+ Assert.assertEquals("Didn't read back text we wrote.", TEXT, result);
+ }
+ }
+
+ @Test
+ public void useWebHDFS() throws IOException, URISyntaxException {
+ try (final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(
+ cluster.getConfiguration(0), WebHdfsConstants.WEBHDFS_SCHEME)) {
+ simpleReadAfterWrite(fs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml
new file mode 100644
index 0000000..cd13532
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <!-- Turn security off for tests by default -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+ <!-- Disable min block size since most tests use tiny blocks -->
+ <property>
+ <name>dfs.namenode.fs-limits.min-block-size</name>
+ <value>0</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..db66160
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
new file mode 100644
index 0000000..8ab70d8
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -0,0 +1,745 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <description>Apache Hadoop Minicluster for Clients</description>
+ <name>Apache Hadoop Client Test Minicluster</name>
+
+ <properties>
+ <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+ <!-- We contain no source -->
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <!-- Leave JUnit as a direct dependency -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <!-- Adding hadoop-annotations so we can make it optional to remove from our transitive tree -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>compile</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- uncomment this dependency if you need to use
+ `mvn dependency:tree -Dverbose` to determine if a dependency shows up
+ in both the hadoop-client-* artifacts and something under minicluster.
+ -->
+<!--
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <!-- Exclude the in-development timeline service and
+ add it as an optional runtime dependency
+ -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ </exclusion>
+ <!-- exclude everything that comes in via the shaded runtime and api TODO remove once we have a filter for "is in these artifacts" -->
+ <!-- Skip jersey, since we need it again here. -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <!-- exclude things that came in via transitive in shaded runtime and api -->
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xmlenc</groupId>
+ <artifactId>xmlenc</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Add optional runtime dependency on the in-development timeline server module
+ to indicate that downstream folks interested in turning it on need that dep.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Add back in transitive dependencies of hadoop-minicluster that are test-jar artifacts excluded as a side effect of excluding the jar
+ Note that all of these must be marked "optional" because they won't be removed from the reduced-dependencies pom after they're included.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ <type>test-jar</type>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ <type>test-jar</type>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>compile</scope>
+ <type>test-jar</type>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Add back in the transitive dependencies excluded from hadoop-common in client TODO remove once we have a filter for "is in these artifacts" -->
+ <!-- skip javax.servlet:servlet-api because it's in client -->
+ <!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
+ <!-- Skip jetty-util because it's in client -->
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jdt</groupId>
+ <artifactId>core</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
+ <dependency>
+ <groupId>net.sf.kosmosfs</groupId>
+ <artifactId>kfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <!-- add back in transitive dependencies of hadoop-mapreduce-client-app removed in client -->
+ <!-- Skipping javax.servlet:servlet-api because it's in client -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- skipping hadoop-annotations -->
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <!-- skipping junit:junit because it is test scope -->
+ <!-- skipping avro because it is in client via hadoop-common -->
+ <!-- skipping jline:jline because it is only present at test scope in the original -->
+ <!-- skipping io.netty:netty because it's in client -->
+ <!-- add back in transitive dependencies of hadoop-yarn-api removed in client -->
+ <!-- skipping hadoop-annotations -->
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <!-- excluding because client already has the tomcat version -->
+ <exclusion>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- skipping jersey-server because it's above -->
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <!-- skipping guice-servlet because it's above -->
+ <!-- skipping avro because it is in client via hadoop-common -->
+ <!-- skipping jersey-core because it's above -->
+ <!-- skipping jersey-json because it's above. -->
+ <!-- skipping io.netty:netty because it's in client -->
+ <!-- Add back in transitive dependencies from hadoop-mapreduce-client-core that were excluded by client -->
+ <!-- skipping junit:junit because it is test scope -->
+ <!-- skipping guice because it's above -->
+ <!-- skipping jersey-test-framework-grizzly2 because it's above -->
+ <!-- skipping jersey-server because it's above -->
+ <!-- skipping jersey-guice because it's above -->
+ <!-- skipping avro because it is in client via hadoop-common -->
+ <!-- skipping hadoop-annotations -->
+ <!-- skipping guice-servlet because it's above -->
+ <!-- skipping jersey-json because it's above. -->
+ <!-- skipping io.netty:netty because it's in client -->
+ <!-- add back in transitive dependencies of hadoop-mapreduce-client-jobclient that were excluded from client -->
+ <!-- skipping junit:junit because it is test scope -->
+ <!-- skipping avro because it is in client via hadoop-common -->
+ <!-- skipping hadoop-annotations -->
+ <!-- skipping guice-servlet because it's above -->
+ <!-- skipping io.netty:netty because it's in client -->
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- We contain no source -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <configuration>
+ <skipSource>true</skipSource>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createSourceJar>true</createSourceJar>
+ <artifactSet>
+ <excludes>
+ <!-- Fine to expose our already-shaded deps as dependencies -->
+ <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+ <exclude>org.apache.hadoop:hadoop-client-api</exclude>
+ <exclude>org.apache.hadoop:hadoop-client-runtime</exclude>
+ <!-- Fine to expose our purposefully not-shaded deps as dependencies -->
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>junit:junit</exclude>
+ <!-- Keep optional runtime deps out of the shading -->
+ <exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice</exclude>
+ <exclude>log4j:log4j</exclude>
+ <!-- We need a filter that matches just those things that are included in the above artiacts -->
+ </excludes>
+ </artifactSet>
+ <filters>
+ <!-- Some of our dependencies include source, so remove it. -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </filter>
+ <!-- We pull in several test jars; keep out the actual test classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/Test*.class</exclude>
+ </excludes>
+ </filter>
+ <!-- Since runtime has classes for these jars, we exclude them.
+ We still want the java services api files, since those were excluded in runtime
+ -->
+ <filter>
+ <artifact>com.sun.jersey:jersey-client</artifact>
+ <excludes>
+ <exclude>**/*.class</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>com.sun.jersey:jersey-core</artifact>
+ <excludes>
+ <exclude>**/*.class</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>com.sun.jersey:jersey-servlet</artifact>
+ <excludes>
+ <exclude>**/*.class</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+ <excludes>
+ <exclude>org/apache/hadoop/*</exclude>
+ <exclude>org/apache/hadoop/**/*</exclude>
+ <!-- Our non-shaded htrace and logging libraries -->
+ <exclude>org/apache/htrace/*</exclude>
+ <exclude>org/apache/htrace/**/*</exclude>
+ <exclude>org/slf4j/*</exclude>
+ <exclude>org/slf4j/**/*</exclude>
+ <exclude>org/apache/commons/logging/*</exclude>
+ <exclude>org/apache/commons/logging/**/*</exclude>
+ <exclude>org/apache/log4j/*</exclude>
+ <exclude>org/apache/log4j/**/*</exclude>
+ <exclude>**/pom.xml</exclude>
+ <!-- Our non-shaded JUnit library -->
+ <exclude>org/junit/*</exclude>
+ <exclude>org/junit/**/*</exclude>
+ <!-- Not the org/ packages that are a part of the jdk -->
+ <exclude>org/ietf/jgss/*</exclude>
+ <exclude>org/omg/**/*</exclude>
+ <exclude>org/w3c/dom/*</exclude>
+ <exclude>org/w3c/dom/**/*</exclude>
+ <exclude>org/xml/sax/*</exclude>
+ <exclude>org/xml/sax/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>com/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Not the com/ packages that are a part of particular jdk implementations -->
+ <exclude>com/sun/tools/*</exclude>
+ <exclude>com/sun/javadoc/*</exclude>
+ <exclude>com/sun/security/*</exclude>
+ <exclude>com/sun/jndi/*</exclude>
+ <exclude>com/sun/management/*</exclude>
+ <exclude>com/sun/tools/**/*</exclude>
+ <exclude>com/sun/javadoc/**/*</exclude>
+ <exclude>com/sun/security/**/*</exclude>
+ <exclude>com/sun/jndi/**/*</exclude>
+ <exclude>com/sun/management/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>io/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>io/compression/*</exclude>
+ <exclude>io/compression/**/*</exclude>
+ <exclude>io/mapfile/*</exclude>
+ <exclude>io/mapfile/**/*</exclude>
+ <exclude>io/map/index/*</exclude>
+ <exclude>io/seqfile/*</exclude>
+ <exclude>io/seqfile/**/*</exclude>
+ <exclude>io/file/buffer/size</exclude>
+ <exclude>io/skip/checksum/errors</exclude>
+ <exclude>io/sort/*</exclude>
+ <exclude>io/serializations</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/el/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/inject/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.inject.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/servlet/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>net/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>net/topology/*</exclude>
+ <exclude>net/topology/**/*</exclude>
+ </excludes>
+ </relocation>
+ </relocations>
+ <transformers>
+ <!-- Needed until MSHADE-182 -->
+ <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70ca1f1e/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
new file mode 100644
index 0000000..aa1a54b
--- /dev/null
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -0,0 +1,318 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+<parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+</parent>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <description>Apache Hadoop Client</description>
+ <name>Apache Hadoop Client Runtime</name>
+
+ <properties>
+ <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+ <!-- We contain no source -->
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <!-- We list this as optional because as a type-pom it won't get included in the shading.
+ Marking it optional means it doesn't count as a transitive dependency of this artifact.
+ -->
+ <optional>true</optional>
+ <exclusions>
+ <!-- these APIs are a part of the SE JDK -->
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xml-apis</groupId>
+ <artifactId>xml-apis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- At runtime anyone using us must have the api present -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <!-- This comes from our parent pom. If we don't expressly change it here to get included,
+ downstream will get warnings at compile time. -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Since hadoop-client is listed as optional, we have to list transitive
+ dependencies that we still want to show up.
+ * HTrace
+ * Slf4j API
+ * commons-logging
+ -->
+ <dependency>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core4</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <!-- Move log4j to optional, since it is needed for some pieces folks might not use:
+ * one of the three custom log4j appenders we have
+ * JobConf (?!) (so essentially any user of MapReduce)
+ -->
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- We contain no source -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <configuration>
+ <skipSource>true</skipSource>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createSourceJar>true</createSourceJar>
+ <artifactSet>
+ <excludes>
+ <!-- We need a filter that matches just those things that aer included in the api jar -->
+ <exclude>org.apache.hadoop:hadoop-client-api</exclude>
+ <!-- Leave HTrace as an unshaded dependency on purpose, since a static class member is used to trace within a given JVM instance -->
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+ <exclude>commons-logging:commons-logging</exclude>
+ <!-- Leave log4j unshaded so downstream users can configure logging. -->
+ <exclude>log4j:log4j</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <!-- We need a filter that matches just those things that are included in the api jar -->
+ <filter>
+ <artifact>org.apache.hadoop:*</artifact>
+ <excludes>
+ <exclude>**/*</exclude>
+ <exclude>*</exclude>
+ </excludes>
+ </filter>
+ <!-- Some of our dependencies include source, so remove it. -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </filter>
+ <!-- We only want one copy of the Localizer class. So long as we keep jasper compiler and runtime on the same version, which one doesn't matter -->
+ <filter>
+ <artifact>tomcat:jasper-compiler</artifact>
+ <excludes>
+ <exclude>org/apache/jasper/compiler/Localizer.class</exclude>
+ </excludes>
+ </filter>
+ <!-- We only have xerces as a dependency for XML output for the fsimage edits, we don't need anything specific to it for javax xml support -->
+ <filter>
+ <artifact>xerces:xercesImpl</artifact>
+ <excludes>
+ <exclude>META-INF/services/*</exclude>
+ </excludes>
+ </filter>
+ <!-- We rely on jersey for our web interfaces. We want to use its java services stuff only internal to jersey -->
+ <filter>
+ <artifact>com.sun.jersey:*</artifact>
+ <excludes>
+ <exclude>META-INF/services/javax.*</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+ <excludes>
+ <exclude>org/apache/hadoop/*</exclude>
+ <exclude>org/apache/hadoop/**/*</exclude>
+ <!-- Our non-shaded htrace and logging libraries -->
+ <exclude>org/apache/htrace/*</exclude>
+ <exclude>org/apache/htrace/**/*</exclude>
+ <exclude>org/slf4j/*</exclude>
+ <exclude>org/slf4j/**/*</exclude>
+ <exclude>org/apache/commons/logging/*</exclude>
+ <exclude>org/apache/commons/logging/**/*</exclude>
+ <exclude>org/apache/log4j/*</exclude>
+ <exclude>org/apache/log4j/**/*</exclude>
+ <exclude>**/pom.xml</exclude>
+ <!-- Not the org/ packages that are a part of the jdk -->
+ <exclude>org/ietf/jgss/*</exclude>
+ <exclude>org/omg/**/*</exclude>
+ <exclude>org/w3c/dom/*</exclude>
+ <exclude>org/w3c/dom/**/*</exclude>
+ <exclude>org/xml/sax/*</exclude>
+ <exclude>org/xml/sax/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>com/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Not the com/ packages that are a part of particular jdk implementations -->
+ <exclude>com/sun/tools/*</exclude>
+ <exclude>com/sun/javadoc/*</exclude>
+ <exclude>com/sun/security/*</exclude>
+ <exclude>com/sun/jndi/*</exclude>
+ <exclude>com/sun/management/*</exclude>
+ <exclude>com/sun/tools/**/*</exclude>
+ <exclude>com/sun/javadoc/**/*</exclude>
+ <exclude>com/sun/security/**/*</exclude>
+ <exclude>com/sun/jndi/**/*</exclude>
+ <exclude>com/sun/management/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>io/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>io/compression/*</exclude>
+ <exclude>io/compression/**/*</exclude>
+ <exclude>io/mapfile/*</exclude>
+ <exclude>io/mapfile/**/*</exclude>
+ <exclude>io/map/index/*</exclude>
+ <exclude>io/seqfile/*</exclude>
+ <exclude>io/seqfile/**/*</exclude>
+ <exclude>io/file/buffer/size</exclude>
+ <exclude>io/skip/checksum/errors</exclude>
+ <exclude>io/sort/*</exclude>
+ <exclude>io/serializations</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/el/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>javax/servlet/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>net/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>net/topology/*</exclude>
+ <exclude>net/topology/**/*</exclude>
+ </excludes>
+ </relocation>
+ <!-- probably not. -->
+<!--
+ <relocation>
+ <pattern>javax/</pattern>
+ <shadedPattern>${shaded.dependency.prefix}.javax.</shadedPattern>
+ <excludes>
+ <exclude>**/pom.xml</exclude>
+ </excludes>
+ </relocation>
+-->
+ </relocations>
+ <transformers>
+ <!-- Needed until MSHADE-182 -->
+ <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
+ <resource>META-INF/jboss-beans.xml</resource>
+ <!-- Add this to enable loading of DTDs
+ <ignoreDtd>false</ignoreDtd>
+ -->
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>META-INF/mailcap.default</resource>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>META-INF/mimetypes.default</resource>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org