You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/11/14 01:28:10 UTC
tez git commit: TEZ_2928. Framework for Hadoop shims. (hitesh)
Repository: tez
Updated Branches:
refs/heads/master 24ba80f81 -> 40e864d14
TEZ_2928. Framework for Hadoop shims. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/40e864d1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/40e864d1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/40e864d1
Branch: refs/heads/master
Commit: 40e864d145458ce94588b90f065204d80ffa1cc7
Parents: 24ba80f
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Nov 13 16:27:47 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Nov 13 16:27:47 2015 -0800
----------------------------------------------------------------------
BUILDING.txt | 3 +
CHANGES.txt | 1 +
hadoop-shim-impls/findbugs-exclude.xml | 16 +++
.../hadoop-shim-2.4/findbugs-exclude.xml | 16 +++
hadoop-shim-impls/hadoop-shim-2.4/pom.xml | 50 ++++++++
.../hadoop/shim/HadoopShim23_24Provider.java | 33 +++++
.../apache/tez/hadoop/shim/HadoopShim24.java | 45 +++++++
...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++
.../shim/TestHadoop23_24ShimProvider.java | 38 ++++++
.../hadoop-shim-2.6/findbugs-exclude.xml | 16 +++
hadoop-shim-impls/hadoop-shim-2.6/pom.xml | 50 ++++++++
.../hadoop/shim/HadoopShim25_26_27Provider.java | 33 +++++
.../apache/tez/hadoop/shim/HadoopShim26.java | 52 ++++++++
...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++
.../shim/TestHadoop25_26_27ShimProvider.java | 39 ++++++
.../hadoop-shim-2.8/findbugs-exclude.xml | 16 +++
hadoop-shim-impls/hadoop-shim-2.8/pom.xml | 50 ++++++++
.../apache/tez/hadoop/shim/HadoopShim28.java | 52 ++++++++
.../tez/hadoop/shim/HadoopShim28Provider.java | 30 +++++
...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++
.../hadoop/shim/TestHadoopShim28Provider.java | 38 ++++++
hadoop-shim-impls/pom.xml | 58 +++++++++
hadoop-shim/findbugs-exclude.xml | 16 +++
hadoop-shim/pom.xml | 46 +++++++
.../tez/hadoop/shim/DefaultHadoopShim.java | 28 +++++
.../org/apache/tez/hadoop/shim/HadoopShim.java | 60 +++++++++
.../tez/hadoop/shim/HadoopShimProvider.java | 38 ++++++
.../tez/hadoop/shim/HadoopShimsLoader.java | 126 +++++++++++++++++++
pom.xml | 7 ++
tez-dag/pom.xml | 4 +
.../java/org/apache/tez/dag/app/AppContext.java | 3 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 10 ++
32 files changed, 1016 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 7e4d39a..241b7a8 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -89,6 +89,9 @@ For recent versions of Hadoop, you can bundle AWS-S3 (2.6.0+) or Azure (2.7.0+)
$ mvn package -Dhadoop.version=${X} -Paws -Pazure
+Tez also has some shims to provide version-specific implementations for various APIs.
+For more details, please refer to https://cwiki.apache.org/confluence/display/TEZ/HadoopShims
+
For building on Windows with the hadoop24 profile enabled:
Tez has a couple of symlinks in its codebase. Windows is a bit difficult with
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7233b03..399c6d5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
TEZ-2679. Admin forms of launch env settings
ALL CHANGES:
+ TEZ-2928. Framework for Hadoop shims.
TEZ-2935. Add MR slow start translation for ShuffleVertexManager
TEZ-2918. Make progress notifications in IOs
TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
new file mode 100644
index 0000000..095c2b1
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-shim-impls</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-shim-2.4</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
new file mode 100644
index 0000000..d94b5bf
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+public class HadoopShim23_24Provider extends HadoopShimProvider {
+
+ @Override
+ public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+ if (majorVersion == 2 && (minorVersion == 3 || minorVersion == 4)) {
+ return new org.apache.tez.hadoop.shim.HadoopShim24();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
new file mode 100644
index 0000000..3f7e56e
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+
+@Private
+public class HadoopShim24 extends HadoopShim {
+
+ @Override
+ public void setHadoopCallerContext(String context) {
+ // Not supported
+ }
+
+ @Override
+ public void clearHadoopCallerContext() {
+ // Not supported
+ }
+
+ @Override
+ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+ // Not supported
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..a675e20
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim23_24Provider
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
new file mode 100644
index 0000000..9bc4435
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoop23_24ShimProvider {
+
+ @Test
+ public void testShimProvider() {
+ HadoopShim23_24Provider provider = new HadoopShim23_24Provider();
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 1));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 5));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 6));
+ Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
new file mode 100644
index 0000000..17f4711
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-shim-impls</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-shim-2.6</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
new file mode 100644
index 0000000..ab013e9
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+public class HadoopShim25_26_27Provider extends HadoopShimProvider {
+
+ @Override
+ public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+ if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) {
+ return new HadoopShim26();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
new file mode 100644
index 0000000..c1622dc
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+
+@Private
+public class HadoopShim26 extends HadoopShim {
+
+ @Override
+ public void setHadoopCallerContext(String context) {
+ // Not supported
+ }
+
+ @Override
+ public void clearHadoopCallerContext() {
+ // Not supported
+ }
+
+ @Override
+ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+ EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
+ Set<String> supportedTypes = new HashSet<String>();
+ for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
+ supportedTypes.add(resourceType.name());
+ }
+ return supportedTypes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..4226f18
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
new file mode 100644
index 0000000..0e6d997
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoop25_26_27ShimProvider {
+
+ @Test
+ public void testShimProvider() {
+ HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider();
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 1));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 3));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 8));
+ Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
new file mode 100644
index 0000000..18d22c5
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-shim-impls</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-shim-2.8</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
new file mode 100644
index 0000000..0c599e4
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+
+public class HadoopShim28 extends HadoopShim {
+
+ final static CallerContext nullCallerContext = new CallerContext.Builder("").build();
+
+ @Override
+ public void setHadoopCallerContext(String context) {
+ CallerContext.setCurrent(new CallerContext.Builder(context).build());
+ }
+
+ @Override
+ public void clearHadoopCallerContext() {
+ CallerContext.setCurrent(nullCallerContext);
+ }
+
+ @Override
+ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+ EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
+ Set<String> supportedTypes = new HashSet<String>();
+ for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
+ supportedTypes.add(resourceType.name());
+ }
+ return supportedTypes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
new file mode 100644
index 0000000..d03a538
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+public class HadoopShim28Provider extends HadoopShimProvider {
+
+ @Override
+ public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+ if (majorVersion == 2 && minorVersion >= 8) {
+ return new HadoopShim28();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..24fe49d
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim28Provider
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
new file mode 100644
index 0000000..df92627
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoopShim28Provider {
+
+ @Test
+ public void testShimProvider() {
+ HadoopShim28Provider provider = new HadoopShim28Provider();
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 7));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 5));
+ Assert.assertNull(provider.createHadoopShim("foo", 2, 6));
+ Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8));
+ Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml
new file mode 100644
index 0000000..6c8f7d2
--- /dev/null
+++ b/hadoop-shim-impls/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tez</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-shim-impls</artifactId>
+ <packaging>pom</packaging>
+
+ <profiles>
+ <profile>
+ <id>hadoop24</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <modules>
+ <module>hadoop-shim-2.4</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>hadoop26</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <modules>
+ <module>hadoop-shim-2.6</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>hadoop28</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <modules>
+ <module>hadoop-shim-2.8</module>
+ </modules>
+ </profile>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim/findbugs-exclude.xml b/hadoop-shim/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim/pom.xml b/hadoop-shim/pom.xml
new file mode 100644
index 0000000..1f70e6f
--- /dev/null
+++ b/hadoop-shim/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tez</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-shim</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
new file mode 100644
index 0000000..c68777c
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+/**
+ * Default Hadoop Shim. For now, it mimics HadoopShim.
+ */
+@Private
+public class DefaultHadoopShim extends HadoopShim {
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
new file mode 100644
index 0000000..7314c2d
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+
+import com.google.common.collect.Sets;
+
+@Public
+@Unstable
+public abstract class HadoopShim {
+
+ /**
+ * Set up Hadoop Caller Context
+ * @param context Context to be set
+ */
+ public void setHadoopCallerContext(String context) {
+ // Nothing to do
+ }
+
+ /**
+ * Clear the Hadoop Caller Context
+ */
+ public void clearHadoopCallerContext() {
+ // Nothing to do
+ }
+
+ public static String CPU_RESOURCE = "CPU";
+ public static String MEMORY_RESOURCE = "MEMORY";
+
+ /**
+ * Extract supported Resource types from the RM's response when the AM registers
+ * @param response ApplicationMasterResponse from RM after registering
+ * @return Set of Resource types that are supported
+ */
+ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
new file mode 100644
index 0000000..728408c
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Public
+@Unstable
+public abstract class HadoopShimProvider {
+
+ /**
+ * Provides a HadoopShim if the version matches the implementation of the Shim
+ * @param hadoopVersion Full version string of hadoop-common's version
+ * @param majorVersion Major version
+ * @param minorVersion Minor version
+ * @return HadoopShim object if there is a match else null
+ */
+ public abstract HadoopShim createHadoopShim(String hadoopVersion, int majorVersion,
+ int minorVersion);
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
new file mode 100644
index 0000000..fc2ec81
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.ServiceLoader;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Private
+public class HadoopShimsLoader {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopShimsLoader.class);
+
+ private static ServiceLoader<HadoopShimProvider> shimLoader =
+ ServiceLoader.load(HadoopShimProvider.class);
+
+ private final HadoopShim currentShim;
+ private final HadoopShimProvider currentShimProvider;
+
+ public HadoopShimsLoader() {
+ String versionStr = VersionInfo.getVersion();
+ Version version = new Version(versionStr);
+ HadoopShim selectedShim = null;
+ HadoopShimProvider selectedShimProvider = null;
+ LOG.info("Trying to locate HadoopShimProvider for "
+ + "hadoopVersion=" + versionStr
+ + ", majorVersion=" + version.majorVersion
+ + ", minorVersion=" + version.minorVersion);
+ synchronized (shimLoader) {
+ for (HadoopShimProvider provider : shimLoader) {
+ LOG.debug("Trying HadoopShimProvider : "
+ + provider.getClass().getName());
+ HadoopShim hadoopShim = null;
+ try {
+ hadoopShim = provider.createHadoopShim(versionStr,
+ version.majorVersion, version.minorVersion);
+
+ if (hadoopShim != null) {
+ selectedShim = hadoopShim;
+ selectedShimProvider = provider;
+ break;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot pick " + provider.getClass().getName()
+ + " as the HadoopShimProvider - returned null hadoop shim");
+ }
+ }
+ } catch (Exception e) {
+ LOG.info("Failed to use " + provider.getClass().getName()
+ + " due to error: ", e);
+ }
+ }
+ if (selectedShim == null) {
+ currentShim = new DefaultHadoopShim();
+ currentShimProvider = null;
+ } else {
+ currentShim = selectedShim;
+ currentShimProvider = selectedShimProvider;
+ }
+ }
+ LOG.info("Picked HadoopShim " + currentShim.getClass().getName()
+ + ", providerName="
+ + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null" )
+ + ", hadoopVersion=" + versionStr
+ + ", majorVersion=" + version.majorVersion
+ + ", minorVersion=" + version.minorVersion);
+
+ }
+
+ public HadoopShim getHadoopShim() {
+ return currentShim;
+ }
+
+ private static class Version {
+
+ int majorVersion = -1;
+ int minorVersion = -1;
+
+ public Version(String versionString) {
+ int index = 0;
+ StringTokenizer tokenizer = new StringTokenizer(versionString, ".-", true);
+ while (tokenizer.hasMoreTokens()) {
+ String token = tokenizer.nextToken();
+ if (token.equals("-")) {
+ break;
+ }
+ if (token.equals(".")) {
+ continue;
+ }
+ try {
+ int i = Integer.parseInt(token);
+ if (index == 0) {
+ majorVersion = i;
+ } else if (index == 1) {
+ minorVersion = i;
+ break;
+ }
+ ++index;
+ } catch (NumberFormatException nfe) {
+ break;
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b8b8a22..a17f714 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,6 +102,11 @@
<dependencies>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-api</artifactId>
<version>${project.version}</version>
</dependency>
@@ -679,6 +684,7 @@
</dependencyManagement>
<modules>
+ <module>hadoop-shim</module>
<module>tez-api</module>
<module>tez-common</module>
<module>tez-runtime-library</module>
@@ -691,6 +697,7 @@
<module>tez-ui</module>
<module>tez-plugins</module>
<module>tez-tools</module>
+ <module>hadoop-shim-impls</module>
<module>tez-dist</module>
<module>docs</module>
</modules>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index adda218..7ca92db 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -35,6 +35,10 @@
<dependencies>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-api</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 657267b..68453b1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -37,6 +37,7 @@ import org.apache.tez.dag.app.rm.node.AMNodeTracker;
import org.apache.tez.common.security.ACLManager;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.hadoop.shim.HadoopShim;
import com.google.common.util.concurrent.ListeningExecutorService;
@@ -123,4 +124,6 @@ public interface AppContext {
public String getTaskSchedulerName(int schedulerId);
public String getContainerLauncherName(int launcherId);
+ public HadoopShim getHadoopShim();
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index f4c57e3..9c09f95 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -174,6 +174,8 @@ import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.Graph;
import org.apache.tez.dag.utils.RelocalizationUtils;
import org.apache.tez.dag.utils.Simple2LevelVersionComparator;
+import org.apache.tez.hadoop.shim.HadoopShim;
+import org.apache.tez.hadoop.shim.HadoopShimsLoader;
import org.apache.tez.util.TezMxBeanResourceCalculator;
import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
@@ -234,6 +236,7 @@ public class DAGAppMaster extends AbstractService {
private final String[] localDirs;
private final String[] logDirs;
private final AMPluginDescriptorProto amPluginDescriptorProto;
+ private final HadoopShim hadoopShim;
private ContainerSignatureMatcher containerSignatureMatcher;
private AMContainerMap containers;
private AMNodeTracker nodes;
@@ -353,9 +356,11 @@ public class DAGAppMaster extends AbstractService {
this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
this.containerID.toString(), this.appMasterUgi.getShortUserName());
+ this.hadoopShim = new HadoopShimsLoader().getHadoopShim();
LOG.info("Created DAGAppMaster for application " + applicationAttemptId
+ ", versionInfo=" + dagVersionInfo.toString());
+
}
// Pull this WebAppUtils function into Tez until YARN-4186
@@ -1596,6 +1601,11 @@ public class DAGAppMaster extends AbstractService {
}
@Override
+ public HadoopShim getHadoopShim() {
+ return hadoopShim;
+ }
+
+ @Override
public Map<ApplicationAccessType, String> getApplicationACLs() {
if (getServiceState() != STATE.STARTED) {
throw new TezUncheckedException(