You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/12/12 01:35:11 UTC
tez git commit: TEZ-2977. Make HadoopShim selection be overridable
for distro-specific implementations. (hitesh)
Repository: tez
Updated Branches:
refs/heads/master 7b16b77aa -> 7940facf6
TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7940facf
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7940facf
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7940facf
Branch: refs/heads/master
Commit: 7940facf676087eab0301d6ada96d0b6576ca693
Parents: 7b16b77
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Dec 11 16:34:16 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Dec 11 16:34:55 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
hadoop-shim-impls/hadoop-shim-2.4/pom.xml | 6 ++
.../shim/TestHadoop23_24ShimProvider.java | 44 ++++++++++++++
hadoop-shim-impls/hadoop-shim-2.6/pom.xml | 6 ++
.../shim/TestHadoop25_26_27ShimProvider.java | 42 +++++++++++++
hadoop-shim-impls/hadoop-shim-2.8/pom.xml | 6 ++
.../hadoop/shim/TestHadoopShim28Provider.java | 42 +++++++++++++
.../tez/hadoop/shim/HadoopShimsLoader.java | 63 +++++++++++++++++++-
.../tez/hadoop/shim/DummyShimProvider.java | 30 ++++++++++
.../tez/hadoop/shim/TestHadoopShimsLoader.java | 54 +++++++++++++++++
hadoop-shim/src/test/resources/log4j.properties | 19 ++++++
pom.xml | 6 ++
tez-dag/findbugs-exclude.xml | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 6 +-
14 files changed, 322 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b19c4be..1e1537f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ INCOMPATIBLE CHANGES
TEZ-604. Revert temporary changes made in TEZ-603 to kill the provided tez session, if running a MapReduce job.
ALL CHANGES:
+ TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations.
TEZ-2472. Change slf4j version to 1.7.10.
TEZ-2920. org.apache.tez.client.TestTezClient.testStopRetriesUntilTimeout is flaky.
TEZ-2824. Add javadocs for Vertex.setConf and DAG.setConf.
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
index 095c2b1..3ffe4e2 100644
--- a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
@@ -38,6 +38,12 @@
<artifactId>hadoop-shim</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
index 9bc4435..871fb67 100644
--- a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
@@ -18,6 +18,8 @@
package org.apache.tez.hadoop.shim;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim;
import org.junit.Assert;
import org.junit.Test;
@@ -33,6 +35,48 @@ public class TestHadoop23_24ShimProvider {
Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4));
+
+ Assert.assertEquals(HadoopShim24.class,
+ provider.createHadoopShim("foo", 2, 3).getClass());
+
+ }
+
+ @Test
+ public void testLoaderOverride() {
+ Configuration conf = new Configuration(false);
+ // Set shim and version to ensure that hadoop version from the build does not create
+ // a mismatch
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim23_24Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.4.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(HadoopShim24.class, shim.getClass());
+ }
+
+ @Test
+ public void testInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that shim in this module does not match
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.6.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+ }
+
+ @Test
+ public void testLoaderOverrideInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that override shim does not return a valid shim
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim23_24Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
index 17f4711..8346c37 100644
--- a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
@@ -38,6 +38,12 @@
<artifactId>hadoop-shim</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
index 0e6d997..13faf37 100644
--- a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
@@ -18,6 +18,7 @@
package org.apache.tez.hadoop.shim;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@@ -34,6 +35,47 @@ public class TestHadoop25_26_27ShimProvider {
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7));
+
+ Assert.assertEquals(HadoopShim26.class,
+ provider.createHadoopShim("foo", 2, 7).getClass());
+ }
+
+ @Test
+ public void testLoaderOverride() {
+ Configuration conf = new Configuration(false);
+ // Set shim and version to ensure that hadoop version from the build does not create
+ // a mismatch
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim25_26_27Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(HadoopShim26.class, shim.getClass());
+ }
+
+ @Test
+ public void testInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that shim in this module does not match
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+ }
+
+ @Test
+ public void testLoaderOverrideInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that override shim does not return a valid shim
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim25_26_27Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
index 18d22c5..41928fe 100644
--- a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
@@ -38,6 +38,12 @@
<artifactId>hadoop-shim</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
index df92627..353d590 100644
--- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
@@ -18,6 +18,7 @@
package org.apache.tez.hadoop.shim;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@@ -33,6 +34,47 @@ public class TestHadoopShim28Provider {
Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8));
Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111));
+
+ Assert.assertEquals(HadoopShim28.class,
+ provider.createHadoopShim("foo", 2, 9).getClass());
+ }
+
+ @Test
+ public void testLoaderOverride() {
+ Configuration conf = new Configuration(false);
+ // Set shim and version to ensure that hadoop version from the build does not create
+ // a mismatch
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim28Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.8.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(HadoopShim28.class, shim.getClass());
+ }
+
+ @Test
+ public void testInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that shim in this module does not match
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.2.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+ }
+
+ @Test
+ public void testLoaderOverrideInvalidVersion() {
+ Configuration conf = new Configuration(false);
+ // Set incompatible version so that override shim does not return a valid shim
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+ HadoopShim28Provider.class.getName());
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
index fc2ec81..9d9dc3f 100644
--- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
@@ -22,10 +22,13 @@ import java.util.ServiceLoader;
import java.util.StringTokenizer;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
@Private
public class HadoopShimsLoader {
@@ -37,17 +40,57 @@ public class HadoopShimsLoader {
private final HadoopShim currentShim;
private final HadoopShimProvider currentShimProvider;
- public HadoopShimsLoader() {
+ public static final String TEZ_HADOOP_SHIM_PROVIDER_CLASS =
+ "tez.hadoop.shim.provider.class";
+
+ // A way to override the hadoop version for testing
+ @Private
+ @VisibleForTesting
+ static final String TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE =
+ "tez.hadoop.shim.hadoop.version.override";
+
+ public HadoopShimsLoader(Configuration conf) {
+ this(conf, false);
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ HadoopShimsLoader(Configuration conf, boolean useReflection) {
+ String overrideProviderClassStr = conf.get(TEZ_HADOOP_SHIM_PROVIDER_CLASS);
+ if (overrideProviderClassStr != null && !overrideProviderClassStr.isEmpty()) {
+ LOG.info("HadoopShim Selection is overridden, using Configured Provider="
+ + overrideProviderClassStr);
+ } else {
+ overrideProviderClassStr = null;
+ }
+
String versionStr = VersionInfo.getVersion();
+ String overrideVersionStr = conf.get(TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE);
+ if (overrideVersionStr != null && !overrideVersionStr.isEmpty()) {
+ LOG.warn("Using overridden hadoop version instead of actual version"
+ + ", realVersion=" + versionStr
+ + ", overrideVersion=" + overrideVersionStr);
+ versionStr = overrideVersionStr;
+ }
+
Version version = new Version(versionStr);
HadoopShim selectedShim = null;
HadoopShimProvider selectedShimProvider = null;
+
LOG.info("Trying to locate HadoopShimProvider for "
+ "hadoopVersion=" + versionStr
+ ", majorVersion=" + version.majorVersion
+ ", minorVersion=" + version.minorVersion);
synchronized (shimLoader) {
for (HadoopShimProvider provider : shimLoader) {
+ if (overrideProviderClassStr != null
+ && !provider.getClass().getName().equals(overrideProviderClassStr)) {
+ LOG.debug("Skipping HadoopShimProvider : "
+ + provider.getClass().getName()
+ + " as config provided to override selection");
+ continue;
+ }
+
LOG.debug("Trying HadoopShimProvider : "
+ provider.getClass().getName());
HadoopShim hadoopShim = null;
@@ -70,6 +113,21 @@ public class HadoopShimsLoader {
+ " due to error: ", e);
}
}
+ if (selectedShim == null && useReflection && overrideProviderClassStr != null) {
+ try {
+ LOG.debug("Using Reflection to create HadoopShim from provider class="
+ + overrideProviderClassStr);
+ Class<HadoopShimProvider> clazz = (Class<HadoopShimProvider>)Class.forName(
+ overrideProviderClassStr, true, Thread.currentThread().getContextClassLoader());
+ selectedShimProvider = clazz.newInstance();
+ selectedShim = selectedShimProvider.createHadoopShim(versionStr,
+ version.majorVersion, version.minorVersion);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to create HadoopShim from provider class: "
+ + overrideProviderClassStr, e);
+ }
+ }
+
if (selectedShim == null) {
currentShim = new DefaultHadoopShim();
currentShimProvider = null;
@@ -80,7 +138,8 @@ public class HadoopShimsLoader {
}
LOG.info("Picked HadoopShim " + currentShim.getClass().getName()
+ ", providerName="
- + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null" )
+ + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null")
+ + ", overrideProviderViaConfig=" + overrideProviderClassStr
+ ", hadoopVersion=" + versionStr
+ ", majorVersion=" + version.majorVersion
+ ", minorVersion=" + version.minorVersion);
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
new file mode 100644
index 0000000..7b72c7a
--- /dev/null
+++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+public class DummyShimProvider extends HadoopShimProvider {
+
+ public static class DummyShim extends HadoopShim {
+ }
+
+ @Override
+ public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+ return new DummyShim();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
new file mode 100644
index 0000000..b0215ed
--- /dev/null
+++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoopShimsLoader {
+
+ @Test
+ public void testBasicLoader() {
+ HadoopShimsLoader loader = new HadoopShimsLoader(new Configuration(false));
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+ }
+
+ @Test
+ public void testLoaderOverride() {
+ Configuration conf = new Configuration(false);
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, DummyShimProvider.class.getName());
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ Assert.assertNotNull(shim);
+ Assert.assertEquals(DummyShim.class, shim.getClass());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testInvalidOverride() {
+ Configuration conf = new Configuration(false);
+ conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, "org.apache.tez.foo");
+ HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+ HadoopShim shim = loader.getHadoopShim();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/resources/log4j.properties b/hadoop-shim/src/test/resources/log4j.properties
new file mode 100644
index 0000000..63353bd
--- /dev/null
+++ b/hadoop-shim/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=debug,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ac449e2..7759de5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,6 +108,12 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-api</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml
index 2122c5e..43046d9 100644
--- a/tez-dag/findbugs-exclude.xml
+++ b/tez-dag/findbugs-exclude.xml
@@ -161,6 +161,7 @@
<Field name="nodes"/>
<Field name="recoveryEnabled"/>
<Field name="isLocal"/>
+ <Field name="hadoopShim"/>
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 05261ba..5cbdab4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -240,7 +240,7 @@ public class DAGAppMaster extends AbstractService {
private final String[] localDirs;
private final String[] logDirs;
private final AMPluginDescriptorProto amPluginDescriptorProto;
- private final HadoopShim hadoopShim;
+ private HadoopShim hadoopShim;
private ContainerSignatureMatcher containerSignatureMatcher;
private AMContainerMap containers;
private AMNodeTracker nodes;
@@ -360,7 +360,6 @@ public class DAGAppMaster extends AbstractService {
this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
this.containerID.toString(), this.appMasterUgi.getShortUserName());
- this.hadoopShim = new HadoopShimsLoader().getHadoopShim();
LOG.info("Created DAGAppMaster for application " + applicationAttemptId
+ ", versionInfo=" + dagVersionInfo.toString());
@@ -417,6 +416,9 @@ public class DAGAppMaster extends AbstractService {
this.amConf = conf;
initResourceCalculatorPlugins();
+ this.hadoopShim = new HadoopShimsLoader(this.amConf).getHadoopShim();
+
+
this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);