You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2016/10/10 11:52:11 UTC
[12/50] [abbrv] oozie git commit: OOZIE-2675 Drop support for hadoop
0.23 (abhishekbafna via rohini)
OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2fd64fa6
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2fd64fa6
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2fd64fa6
Branch: refs/heads/oya
Commit: 2fd64fa646708ae684f2274729a0df6623598709
Parents: 9dc474e
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Mon Sep 19 09:53:25 2016 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Mon Sep 19 09:53:25 2016 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/HadoopELFunctions.java | 5 --
.../oozie/command/wf/SubmitMRXCommand.java | 3 --
.../java/org/apache/oozie/util/JobUtils.java | 2 +-
.../java/org/apache/oozie/test/XTestCase.java | 4 +-
docs/src/site/twiki/DG_QuickStart.twiki | 5 +-
.../site/twiki/DG_SqoopActionExtension.twiki | 2 +-
docs/src/site/twiki/ENG_Building.twiki | 5 +-
hadooplibs/hadoop-auth-0.23/pom.xml | 45 -----------------
hadooplibs/hadoop-distcp-0.23/pom.xml | 43 -----------------
hadooplibs/hadoop-utils-0.23/pom.xml | 42 ----------------
.../action/hadoop/LauncherMainHadoopUtils.java | 35 --------------
.../apache/oozie/hadoop/utils/HadoopShims.java | 51 --------------------
hadooplibs/pom.xml | 11 -----
pom.xml | 12 -----
release-log.txt | 1 +
15 files changed, 8 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java
index c322887..ad2a71d 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java
@@ -52,11 +52,6 @@ public class HadoopELFunctions {
Map<String, Map<String, Long>> counters = (Map<String, Map<String, Long>>) obj;
if (counters == null) {
counters = getCounters(nodeName);
- // In Hadoop 0.23 they deprecated 'org.apache.hadoop.mapred.Task$Counter' and they REMOVED IT
- // Here we are getting the new Name and inserting it using the old name if the old name is not found
- if (counters.get(RECORDS) == null) {
- counters.put(RECORDS, counters.get(RECORDS_023));
- }
instance.setTransientVar(nodeName + WorkflowInstance.NODE_VAR_SEPARATOR + HADOOP_COUNTERS, counters);
}
return counters;
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
index 9124a45..cc61d3d 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
@@ -43,9 +43,6 @@ public class SubmitMRXCommand extends SubmitHttpXCommand {
SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER);
SKIPPED_CONFS.add(XOozieClient.JT);
SKIPPED_CONFS.add(XOozieClient.NN);
- // a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS'
- // in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1
- SKIPPED_CONFS.add(XOozieClient.NN_2);
DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2);
DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2);
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/util/JobUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java
index a4d7272..a7a53b3 100644
--- a/core/src/main/java/org/apache/oozie/util/JobUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java
@@ -162,7 +162,7 @@ public class JobUtils {
URI uri = fs.makeQualified(file).toUri();
DistributedCache.addCacheFile(uri, conf);
}
- else { // Hadoop 0.23/2.x
+ else { // Hadoop 2.x
DistributedCache.addFileToClassPath(file, conf, fs);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index 8ce44f3..3dd99d7 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -967,9 +967,7 @@ public abstract class XTestCase extends TestCase {
conf.set("dfs.permissions", "true");
conf.set("hadoop.security.authentication", "simple");
- //Doing this because Hadoop 1.x does not support '*' and
- //Hadoop 0.23.x does not process wildcard if the value is
- // '*,127.0.0.1'
+ //Doing this because Hadoop 1.x does not support '*' if the value is '*,127.0.0.1'
StringBuilder sb = new StringBuilder();
sb.append("127.0.0.1,localhost");
for (InetAddress i : InetAddress.getAllByName(InetAddress.getLocalHost().getHostName())) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index 2cfaa51..0653f08 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -41,10 +41,9 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe
2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are
available to customise the versions of the dependencies:
--P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop
+-P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop
profile depending on the hadoop version used.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT
- for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/DG_SqoopActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_SqoopActionExtension.twiki b/docs/src/site/twiki/DG_SqoopActionExtension.twiki
index 1e7f2a8..906ba54 100644
--- a/docs/src/site/twiki/DG_SqoopActionExtension.twiki
+++ b/docs/src/site/twiki/DG_SqoopActionExtension.twiki
@@ -10,7 +10,7 @@
---++ Sqoop Action
-*IMPORTANT:* The Sqoop action requires Apache Hadoop 0.23.
+*IMPORTANT:* The Sqoop action requires Apache Hadoop 1.x or 2.x.
The =sqoop= action runs a Sqoop job.
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index d138817..9864098 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -212,10 +212,9 @@ $ bin/mkdistro.sh [-DskipTests]
Running =mkdistro.sh= will create the binary distribution of Oozie. The following options are available to customise
the versions of the dependencies:
-Puber - Bundle required hadoop and hcatalog libraries in oozie war
--P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop
+-P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop
profile depending on the hadoop version used.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0
-.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-auth-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-auth-0.23/pom.xml b/hadooplibs/hadoop-auth-0.23/pom.xml
deleted file mode 100644
index 7b3b466..0000000
--- a/hadooplibs/hadoop-auth-0.23/pom.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Auth</description>
- <name>Apache Oozie Hadoop Auth ${project.version} Test</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-distcp-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-distcp-0.23/pom.xml b/hadooplibs/hadoop-distcp-0.23/pom.xml
deleted file mode 100644
index 60c8e02..0000000
--- a/hadooplibs/hadoop-distcp-0.23/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Distcp ${project.version}</description>
- <name>Apache Oozie Hadoop Distcp ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/pom.xml b/hadooplibs/hadoop-utils-0.23/pom.xml
deleted file mode 100644
index a55e647..0000000
--- a/hadooplibs/hadoop-utils-0.23/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Utils</description>
- <name>Apache Oozie Hadoop Utils</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
deleted file mode 100644
index dca7820..0000000
--- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.conf.Configuration;
-
-
-public class LauncherMainHadoopUtils {
-
- public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
- public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
-
- private LauncherMainHadoopUtils() {
- }
-
- public static void killChildYarnJobs(Configuration actionConf) {
- // no-op
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
deleted file mode 100644
index 799dffb..0000000
--- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.hadoop.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import java.io.IOException;
-
-public class HadoopShims {
- FileSystem fs;
-
- public HadoopShims(FileSystem fs) {
- this.fs = fs;
- }
-
- public static boolean isSymlinkSupported() {
- return false;
- }
-
- public Path getSymLinkTarget(Path p) throws IOException {
- return p;
- }
-
- public boolean isSymlink(Path p) throws IOException {
- return false;
- }
-
- public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- }
-
- public static boolean isYARN() {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml
index 17f1f07..cd0f478 100644
--- a/hadooplibs/pom.xml
+++ b/hadooplibs/pom.xml
@@ -56,17 +56,6 @@
</modules>
</profile>
<profile>
- <id>hadoop-0.23</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-utils-0.23</module>
- <module>hadoop-distcp-0.23</module>
- <module>hadoop-auth-0.23</module>
- </modules>
- </profile>
- <profile>
<id>hadoop-2</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5a8e5f5..2cbc91f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1797,18 +1797,6 @@
</properties>
</profile>
<profile>
- <id>hadoop-0.23</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hadoop.version>0.23.5</hadoop.version>
- <hadoop.majorversion>0.23</hadoop.majorversion>
- <pig.classifier>h2</pig.classifier>
- <sqoop.classifier>hadoop23</sqoop.classifier>
- </properties>
- </profile>
- <profile>
<id>hadoop-2</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d1e36be..681e5ee 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini)
OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter)
OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter)