You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/06/16 20:38:01 UTC
[1/2] oozie git commit: OOZIE-2316 Drop support for Hadoop 1 and 0.23
(asasvari via rkanter)
Repository: oozie
Updated Branches:
refs/heads/pya [created] a37835fec
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
deleted file mode 100644
index dca7820..0000000
--- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.conf.Configuration;
-
-
-public class LauncherMainHadoopUtils {
-
- public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
- public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
-
- private LauncherMainHadoopUtils() {
- }
-
- public static void killChildYarnJobs(Configuration actionConf) {
- // no-op
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
deleted file mode 100644
index 799dffb..0000000
--- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.hadoop.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import java.io.IOException;
-
-public class HadoopShims {
- FileSystem fs;
-
- public HadoopShims(FileSystem fs) {
- this.fs = fs;
- }
-
- public static boolean isSymlinkSupported() {
- return false;
- }
-
- public Path getSymLinkTarget(Path p) throws IOException {
- return p;
- }
-
- public boolean isSymlink(Path p) throws IOException {
- return false;
- }
-
- public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- }
-
- public static boolean isYARN() {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/pom.xml b/hadooplibs/hadoop-utils-1/pom.xml
deleted file mode 100644
index 26f1ed0..0000000
--- a/hadooplibs/hadoop-utils-1/pom.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>hadoop-1-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Utils</description>
- <name>Apache Oozie Hadoop Utils</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
deleted file mode 100644
index dca7820..0000000
--- a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.conf.Configuration;
-
-
-public class LauncherMainHadoopUtils {
-
- public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
- public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
-
- private LauncherMainHadoopUtils() {
- }
-
- public static void killChildYarnJobs(Configuration actionConf) {
- // no-op
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
deleted file mode 100644
index 799dffb..0000000
--- a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.hadoop.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import java.io.IOException;
-
-public class HadoopShims {
- FileSystem fs;
-
- public HadoopShims(FileSystem fs) {
- this.fs = fs;
- }
-
- public static boolean isSymlinkSupported() {
- return false;
- }
-
- public Path getSymLinkTarget(Path p) throws IOException {
- return p;
- }
-
- public boolean isSymlink(Path p) throws IOException {
- return false;
- }
-
- public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- }
-
- public static boolean isYARN() {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/pom.xml b/hadooplibs/hadoop-utils-2/pom.xml
deleted file mode 100644
index a20426d..0000000
--- a/hadooplibs/hadoop-utils-2/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>hadoop-2-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Utils ${project.version}</description>
- <name>Apache Oozie Hadoop Utils ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
deleted file mode 100644
index ce8c14f..0000000
--- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import java.io.IOException;
-import java.lang.String;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-public class LauncherMainHadoopUtils {
-
- public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
- public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
-
- private LauncherMainHadoopUtils() {
- }
-
- private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) {
- System.out.println("Fetching child yarn jobs");
- Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
- String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS);
- if (tag == null) {
- System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS);
- return childYarnJobs;
- }
- System.out.println("tag id : " + tag);
- long startTime = 0L;
- try {
- startTime = Long.parseLong(System.getProperty(OOZIE_JOB_LAUNCH_TIME));
- } catch(NumberFormatException nfe) {
- throw new RuntimeException("Could not find Oozie job launch time", nfe);
- }
-
- GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
- gar.setScope(ApplicationsRequestScope.OWN);
- gar.setApplicationTags(Collections.singleton(tag));
- long endTime = System.currentTimeMillis();
- if (startTime > endTime) {
- System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " +
- "Attempting to work around...");
- // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an
- // offset in both directions
- long diff = 2 * (startTime - endTime);
- startTime = startTime - diff;
- endTime = endTime + diff;
- }
- gar.setStartRange(startTime, endTime);
- try {
- ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
- GetApplicationsResponse apps = proxy.getApplications(gar);
- List<ApplicationReport> appsList = apps.getApplicationList();
- for(ApplicationReport appReport : appsList) {
- childYarnJobs.add(appReport.getApplicationId());
- }
- } catch (IOException ioe) {
- throw new RuntimeException("Exception occurred while finding child jobs", ioe);
- } catch (YarnException ye) {
- throw new RuntimeException("Exception occurred while finding child jobs", ye);
- }
-
- System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ","));
- return childYarnJobs;
- }
-
- public static void killChildYarnJobs(Configuration actionConf) {
- try {
- Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
- if (!childYarnJobs.isEmpty()) {
- System.out.println();
- System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
- System.out.println("Killing existing jobs and starting over:");
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(actionConf);
- yarnClient.start();
- for (ApplicationId app : childYarnJobs) {
- System.out.print("Killing job [" + app + "] ... ");
- yarnClient.killApplication(app);
- System.out.println("Done");
- }
- System.out.println();
- }
- } catch (YarnException ye) {
- throw new RuntimeException("Exception occurred while killing child job(s)", ye);
- } catch (IOException ioe) {
- throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
deleted file mode 100644
index c21884c..0000000
--- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.hadoop.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-import java.net.URI;
-
-public class HadoopShims {
- FileSystem fs;
-
- public HadoopShims(FileSystem fs) {
- this.fs = fs;
- }
-
- public static boolean isSymlinkSupported() {
- return true;
- }
-
- public Path getSymLinkTarget(Path p) throws IOException {
- try {
- //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink
- Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
- return fs.getFileLinkStatus(tempPath).getSymlink();
- }
- catch (java.net.URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public boolean isSymlink(Path p) throws IOException {
- try {
- //isSymlink doesn't work with fragment name, need to remove fragment before checking for symlink
- Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
- return fs.getFileLinkStatus(tempPath).isSymlink();
- }
- catch (java.net.URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- fs.createSymlink(target, link, createParent);
- }
-
- public static boolean isYARN() {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/pom.xml b/hadooplibs/hadoop-utils-3/pom.xml
deleted file mode 100644
index 4013164..0000000
--- a/hadooplibs/hadoop-utils-3/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>hadoop-3-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Utils ${project.version}</description>
- <name>Apache Oozie Hadoop Utils ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
deleted file mode 100644
index 94e01ea..0000000
--- a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-public class LauncherMainHadoopUtils {
-
- public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
- public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
-
- private LauncherMainHadoopUtils() {
- }
-
- private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) {
- System.out.println("Fetching child yarn jobs");
- Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
- String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS);
- if (tag == null) {
- System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS);
- return childYarnJobs;
- }
- System.out.println("tag id : " + tag);
- long startTime = 0L;
- try {
- startTime = Long.parseLong((System.getProperty(OOZIE_JOB_LAUNCH_TIME)));
- } catch(NumberFormatException nfe) {
- throw new RuntimeException("Could not find Oozie job launch time", nfe);
- }
-
- GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
- gar.setScope(ApplicationsRequestScope.OWN);
- gar.setApplicationTags(Collections.singleton(tag));
- long endTime = System.currentTimeMillis();
- if (startTime > endTime) {
- System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " +
- "Attempting to work around...");
- // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an
- // offset in both directions
- long diff = 2 * (startTime - endTime);
- startTime = startTime - diff;
- endTime = endTime + diff;
- }
- gar.setStartRange(startTime, endTime);
- try {
- ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
- GetApplicationsResponse apps = proxy.getApplications(gar);
- List<ApplicationReport> appsList = apps.getApplicationList();
- for(ApplicationReport appReport : appsList) {
- childYarnJobs.add(appReport.getApplicationId());
- }
- } catch (IOException ioe) {
- throw new RuntimeException("Exception occurred while finding child jobs", ioe);
- } catch (YarnException ye) {
- throw new RuntimeException("Exception occurred while finding child jobs", ye);
- }
-
- System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ","));
- return childYarnJobs;
- }
-
- public static void killChildYarnJobs(Configuration actionConf) {
- try {
- Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
- if (!childYarnJobs.isEmpty()) {
- System.out.println();
- System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
- System.out.println("Killing existing jobs and starting over:");
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(actionConf);
- yarnClient.start();
- for (ApplicationId app : childYarnJobs) {
- System.out.print("Killing job [" + app + "] ... ");
- yarnClient.killApplication(app);
- System.out.println("Done");
- }
- System.out.println();
- }
- } catch (YarnException ye) {
- throw new RuntimeException("Exception occurred while killing child job(s)", ye);
- } catch (IOException ioe) {
- throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
deleted file mode 100644
index c21884c..0000000
--- a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.hadoop.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-import java.net.URI;
-
-public class HadoopShims {
- FileSystem fs;
-
- public HadoopShims(FileSystem fs) {
- this.fs = fs;
- }
-
- public static boolean isSymlinkSupported() {
- return true;
- }
-
- public Path getSymLinkTarget(Path p) throws IOException {
- try {
- //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink
- Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
- return fs.getFileLinkStatus(tempPath).getSymlink();
- }
- catch (java.net.URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public boolean isSymlink(Path p) throws IOException {
- try {
- //isSymlink doesn't work with fragment name, need to remove fragment before checking for symlink
- Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
- return fs.getFileLinkStatus(tempPath).isSymlink();
- }
- catch (java.net.URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- fs.createSymlink(target, link, createParent);
- }
-
- public static boolean isYARN() {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml
deleted file mode 100644
index 17f1f07..0000000
--- a/hadooplibs/pom.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadooplibs</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Libs</description>
- <name>Apache Oozie Hadoop Libs</name>
- <packaging>pom</packaging>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-utils-1</module>
- <module>hadoop-distcp-1</module>
- <module>hadoop-auth-1</module>
- </modules>
- </profile>
- <profile>
- <id>hadoop-0.23</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-utils-0.23</module>
- <module>hadoop-distcp-0.23</module>
- <module>hadoop-auth-0.23</module>
- </modules>
- </profile>
- <profile>
- <id>hadoop-2</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-utils-2</module>
- <module>hadoop-distcp-2</module>
- <module>hadoop-auth-2</module>
- </modules>
- </profile>
- <profile>
- <id>hadoop-3</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-distcp-3</module>
- <module>hadoop-auth-3</module>
- <module>hadoop-utils-3</module>
- </modules>
- </profile>
- </profiles>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5b7a863..9d86e71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -76,26 +76,23 @@
</oozie.test.default.config.file>
<oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file>
- <hadoop.version>1.2.1</hadoop.version>
- <hadoop.majorversion>1</hadoop.majorversion>
- <hadooplib.version>hadoop-${hadoop.majorversion}-${project.version}</hadooplib.version>
- <oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version>
+ <hadoop.version>2.4.0</hadoop.version>
<hbase.version>0.94.27</hbase.version>
<dropwizard.metrics.version>3.1.0</dropwizard.metrics.version>
<clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license>
+
<!-- Sharelib component versions -->
<hive.version>0.13.1</hive.version>
<pig.version>0.12.1</pig.version>
- <pig.classifier></pig.classifier>
+ <pig.classifier>h2</pig.classifier>
<sqoop.version>1.4.3</sqoop.version>
<spark.version>1.6.1</spark.version>
<spark.guava.version>14.0.1</spark.guava.version>
- <sqoop.classifier>hadoop100</sqoop.classifier>
+ <sqoop.classifier>hadoop200</sqoop.classifier>
<streaming.version>${hadoop.version}</streaming.version>
- <distcp.version>${hadoop.version}</distcp.version>
<hadoop.auth.version>${hadoop.version}</hadoop.auth.version>
<!-- Tomcat version -->
@@ -104,12 +101,11 @@
<openjpa.version>2.2.2</openjpa.version>
<xerces.version>2.10.0</xerces.version>
<curator.version>2.5.0</curator.version>
- <jackson.version>1.8.8</jackson.version>
+ <jackson.version>1.9.13</jackson.version>
<log4j.version>1.2.17</log4j.version>
</properties>
<modules>
- <module>hadooplibs</module>
<module>client</module>
<module>core</module>
<module>sharelib</module>
@@ -311,13 +307,6 @@
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
- <version>${hadooplib.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.auth.version}</version>
@@ -430,7 +419,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
- <version>${distcp.version}</version>
+ <version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
@@ -443,7 +432,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
- <version>${distcp.version}</version>
+ <version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
@@ -454,13 +443,6 @@
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>${hadooplib.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-webhcat-java-client</artifactId>
<version>${hive.version}</version>
@@ -1452,12 +1434,6 @@
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>${oozie.hadoop-utils.version}</version>
- </dependency>
-
- <dependency>
<groupId>dk.brics.automaton</groupId>
<artifactId>automaton</artifactId>
<version>1.11-8</version>
@@ -1572,6 +1548,7 @@
<plugin>
<groupId>com.atlassian.maven.plugins</groupId>
<artifactId>maven-clover2-plugin</artifactId>
+ <version>4.0.6</version>
<configuration>
<licenseLocation>${clover.license}</licenseLocation>
<generateXml>true</generateXml>
@@ -1585,6 +1562,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.1</version>
<configuration>
<xmlOutput>true</xmlOutput>
<findbugsXmlOutput>true</findbugsXmlOutput>
@@ -1770,57 +1748,9 @@
</plugin>
</plugins>
</build>
+
</profile>
- <profile>
- <id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <hadoop.version>1.2.1</hadoop.version>
- <hadoop.majorversion>1</hadoop.majorversion>
- <pig.classifier></pig.classifier>
- <sqoop.classifier>hadoop100</sqoop.classifier>
- </properties>
- </profile>
- <profile>
- <id>hadoop-0.23</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hadoop.version>0.23.5</hadoop.version>
- <hadoop.majorversion>0.23</hadoop.majorversion>
- <pig.classifier>h2</pig.classifier>
- <sqoop.classifier>hadoop23</sqoop.classifier>
- </properties>
- </profile>
- <profile>
- <id>hadoop-2</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hadoop.version>2.4.0</hadoop.version>
- <hadoop.majorversion>2</hadoop.majorversion>
- <pig.classifier>h2</pig.classifier>
- <sqoop.classifier>hadoop200</sqoop.classifier>
- <jackson.version>1.9.13</jackson.version>
- </properties>
- </profile>
- <profile>
- <id>hadoop-3</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
- <hadoop.majorversion>3</hadoop.majorversion>
- <pig.classifier>h2</pig.classifier>
- <sqoop.classifier>hadoop200</sqoop.classifier>
- <jackson.version>1.9.13</jackson.version>
- </properties>
- </profile>
+
<profile>
<id>loginServerExample</id>
<activation>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 189ca21..3a23cbf 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter)
OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru)
OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru)
OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru)
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/distcp/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/distcp/pom.xml b/sharelib/distcp/pom.xml
index 0b56f07..c8cc47c 100644
--- a/sharelib/distcp/pom.xml
+++ b/sharelib/distcp/pom.xml
@@ -39,8 +39,8 @@
<dependencies>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
<scope>compile</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
index 6ac5ad6..0fb8e8b 100644
--- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
+++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
@@ -48,7 +48,7 @@ public class DistcpMain extends JavaMain {
protected void run(String[] args) throws Exception {
Configuration actionConf = loadActionConf();
- LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ LauncherMain.killChildYarnJobs(actionConf);
String logFile = setUpDistcpLog4J(actionConf);
Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS,
org.apache.hadoop.tools.DistCp.class);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml
index 73802da..d10d7b8 100644
--- a/sharelib/hive/pom.xml
+++ b/sharelib/hive/pom.xml
@@ -122,18 +122,10 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
-
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -150,8 +142,8 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index cb0172d..69c1012 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -294,7 +294,7 @@ public class HiveMain extends LauncherMain {
}
System.out.println();
- LauncherMainHadoopUtils.killChildYarnJobs(hiveConf);
+ LauncherMain.killChildYarnJobs(hiveConf);
System.out.println("=================================================================");
System.out.println();
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive2/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml
index d660b90..ce967c5 100644
--- a/sharelib/hive2/pom.xml
+++ b/sharelib/hive2/pom.xml
@@ -115,11 +115,7 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <scope>provided</scope>
- </dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index e122608..f783e6f 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -228,7 +228,7 @@ public class Hive2Main extends LauncherMain {
}
System.out.println();
- LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ LauncherMain.killChildYarnJobs(actionConf);
System.out.println("=================================================================");
System.out.println();
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml
index 3fa2872..dd95b45 100644
--- a/sharelib/oozie/pom.xml
+++ b/sharelib/oozie/pom.xml
@@ -60,12 +60,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <scope>compile</scope>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
index 10a1b12..32c7434 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
@@ -42,7 +42,7 @@ public class JavaMain extends LauncherMain {
setYarnTag(actionConf);
- LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ LauncherMain.killChildYarnJobs(actionConf);
Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class);
System.out.println("Main class : " + klass.getName());
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index fb190d6..fef6523 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -28,8 +28,12 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -38,11 +42,22 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
public abstract class LauncherMain {
public static final String HADOOP_JOBS = "hadoopJobs";
public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags";
+ public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags";
+ public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time";
protected static void run(Class<? extends LauncherMain> klass, String[] args) throws Exception {
LauncherMain main = klass.newInstance();
@@ -104,6 +119,77 @@ public abstract class LauncherMain {
}
}
+ private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) {
+ System.out.println("Fetching child yarn jobs");
+ Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
+ String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS);
+ if (tag == null) {
+ System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS);
+ return childYarnJobs;
+ }
+ System.out.println("tag id : " + tag);
+ long startTime = 0L;
+ try {
+ startTime = Long.parseLong(System.getProperty(OOZIE_JOB_LAUNCH_TIME));
+ } catch(NumberFormatException nfe) {
+ throw new RuntimeException("Could not find Oozie job launch time", nfe);
+ }
+
+ GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
+ gar.setScope(ApplicationsRequestScope.OWN);
+ gar.setApplicationTags(Collections.singleton(tag));
+ long endTime = System.currentTimeMillis();
+ if (startTime > endTime) {
+ System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " +
+ "Attempting to work around...");
+ // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an
+ // offset in both directions
+ long diff = 2 * (startTime - endTime);
+ startTime = startTime - diff;
+ endTime = endTime + diff;
+ }
+ gar.setStartRange(startTime, endTime);
+ try {
+ ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
+ GetApplicationsResponse apps = proxy.getApplications(gar);
+ List<ApplicationReport> appsList = apps.getApplicationList();
+ for(ApplicationReport appReport : appsList) {
+ childYarnJobs.add(appReport.getApplicationId());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ioe);
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ye);
+ }
+
+ System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ","));
+ return childYarnJobs;
+ }
+
+ public static void killChildYarnJobs(Configuration actionConf) {
+ try {
+ Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
+ if (!childYarnJobs.isEmpty()) {
+ System.out.println();
+ System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
+ System.out.println("Killing existing jobs and starting over:");
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(actionConf);
+ yarnClient.start();
+ for (ApplicationId app : childYarnJobs) {
+ System.out.print("Killing job [" + app + "] ... ");
+ yarnClient.killApplication(app);
+ System.out.println("Done");
+ }
+ System.out.println();
+ }
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ye);
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
+ }
+ }
+
protected abstract void run(String[] args) throws Exception;
/**
@@ -202,13 +288,13 @@ public abstract class LauncherMain {
}
protected static void setYarnTag(Configuration actionConf) {
- if(actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS) != null) {
+ if(actionConf.get(CHILD_MAPREDUCE_JOB_TAGS) != null) {
// in case the user set their own tags, appending the launcher tag.
if(actionConf.get(MAPREDUCE_JOB_TAGS) != null) {
actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(MAPREDUCE_JOB_TAGS) + ","
- + actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS));
+ + actionConf.get(CHILD_MAPREDUCE_JOB_TAGS));
} else {
- actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS));
+ actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(CHILD_MAPREDUCE_JOB_TAGS));
}
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index 545f8e2..f2cba13 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -28,8 +28,6 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.io.OutputStream;
-import java.io.FileOutputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.security.Permission;
@@ -349,9 +347,9 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
// loading action conf prepared by Oozie
Configuration actionConf = LauncherMain.loadActionConf();
- if(actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS) != null) {
+ if(actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS) != null) {
propagationConf.set(LauncherMain.MAPREDUCE_JOB_TAGS,
- actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS));
+ actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS));
}
propagationConf.writeXml(new FileWriter(PROPAGATION_CONF_XML));
@@ -470,9 +468,9 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath());
System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath());
System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath());
- if (getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME) != null) {
- System.setProperty(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME,
- getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME));
+ if (getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) {
+ System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME,
+ getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME));
}
String actionConfigClass = getJobConf().get(OOZIE_ACTION_CONFIG_CLASS);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 23447cf..96d59b9 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -50,7 +50,7 @@ public class MapReduceMain extends LauncherMain {
JobConf jobConf = new JobConf();
addActionConf(jobConf, actionConf);
- LauncherMainHadoopUtils.killChildYarnJobs(jobConf);
+ LauncherMain.killChildYarnJobs(jobConf);
// Run a config class if given to update the job conf
runConfigClass(jobConf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/pig/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml
index bb3eea5..562c530 100644
--- a/sharelib/pig/pom.xml
+++ b/sharelib/pig/pom.xml
@@ -83,12 +83,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
@@ -103,8 +97,8 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 7ba22f1..3d1110b 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -214,7 +214,7 @@ public class PigMain extends LauncherMain {
System.out.println(" " + arg);
}
- LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ LauncherMain.killChildYarnJobs(actionConf);
System.out.println("=================================================================");
System.out.println();
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/spark/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/spark/pom.xml b/sharelib/spark/pom.xml
index 748ae06..46c6375 100644
--- a/sharelib/spark/pom.xml
+++ b/sharelib/spark/pom.xml
@@ -248,6 +248,103 @@
<artifactId>oozie-examples</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-beeline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.spark-project.hive</groupId>
+ <artifactId>hive-shims</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
@@ -320,109 +417,5 @@
</plugin>
</plugins>
</build>
-
- <profiles>
- <profile>
- <id>hadoop-2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_2.10</artifactId>
- <version>${spark.version}</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-app</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-beeline</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-service</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.spark-project.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 0e6e271..9dbd12f 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -65,7 +65,7 @@ public class SparkMain extends LauncherMain {
boolean isPyspark = false;
Configuration actionConf = loadActionConf();
setYarnTag(actionConf);
- LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ LauncherMain.killChildYarnJobs(actionConf);
String logFile = setUpSparkLog4J(actionConf);
List<String> sparkArgs = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/sqoop/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/pom.xml b/sharelib/sqoop/pom.xml
index 5b80a4f..d875c93 100644
--- a/sharelib/sqoop/pom.xml
+++ b/sharelib/sqoop/pom.xml
@@ -214,11 +214,6 @@
<artifactId>oozie-sharelib-oozie</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <scope>provided</scope>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 360cf1e..1f66c4a 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -167,7 +167,7 @@ public class SqoopMain extends LauncherMain {
System.out.println(" " + arg);
}
- LauncherMainHadoopUtils.killChildYarnJobs(sqoopConf);
+ LauncherMain.killChildYarnJobs(sqoopConf);
System.out.println("=================================================================");
System.out.println();
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 7e73572..d0c1806 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -37,8 +37,8 @@
<artifactId>oozie-client</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -271,8 +271,8 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
[2/2] oozie git commit: OOZIE-2316 Drop support for Hadoop 1 and 0.23
(asasvari via rkanter)
Posted by rk...@apache.org.
OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a37835fe
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a37835fe
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a37835fe
Branch: refs/heads/pya
Commit: a37835feced8d4f5c7b53bcf12cc9355de482a8a
Parents: fc9aeb1
Author: Robert Kanter <rk...@cloudera.com>
Authored: Thu Jun 16 13:37:11 2016 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Thu Jun 16 13:37:11 2016 -0700
----------------------------------------------------------------------
client/pom.xml | 4 +-
.../org/apache/oozie/client/XOozieClient.java | 20 +-
core/pom.xml | 4 +-
.../oozie/action/hadoop/JavaActionExecutor.java | 24 +--
.../action/hadoop/LauncherMapperHelper.java | 5 +-
.../action/hadoop/SparkActionExecutor.java | 8 +-
.../oozie/command/wf/SubmitHttpXCommand.java | 2 +-
.../oozie/command/wf/SubmitMRXCommand.java | 9 +-
.../wf/SubmitScriptLanguageXCommand.java | 2 +-
.../oozie/command/wf/SubmitSqoopXCommand.java | 2 +-
.../oozie/service/HadoopAccessorService.java | 9 +-
.../apache/oozie/service/ShareLibService.java | 30 ++-
.../java/org/apache/oozie/util/FSUtils.java | 53 +++++
.../java/org/apache/oozie/util/JobUtils.java | 21 +-
.../action/hadoop/TestFsActionExecutor.java | 4 +-
.../action/hadoop/TestJavaActionExecutor.java | 55 ++---
.../org/apache/oozie/client/TestOozieCLI.java | 5 +-
.../oozie/client/TestWorkflowXClient.java | 10 +-
.../command/wf/TestSubmitHiveXCommand.java | 2 +-
.../oozie/command/wf/TestSubmitMRXCommand.java | 8 +-
.../oozie/command/wf/TestSubmitPigXCommand.java | 6 +-
.../command/wf/TestSubmitSqoopXCommand.java | 2 +-
.../service/TestHadoopAccessorService.java | 2 +-
.../oozie/service/TestShareLibService.java | 43 ++--
.../lite/TestLiteWorkflowAppParser.java | 1 -
distro/src/main/bin/addtowar.sh | 20 +-
docs/src/site/twiki/DG_QuickStart.twiki | 11 +-
docs/src/site/twiki/ENG_Building.twiki | 11 +-
hadooplibs/hadoop-auth-0.23/pom.xml | 45 -----
hadooplibs/hadoop-auth-1/pom.xml | 43 ----
hadooplibs/hadoop-auth-2/pom.xml | 43 ----
hadooplibs/hadoop-distcp-0.23/pom.xml | 43 ----
hadooplibs/hadoop-distcp-1/pom.xml | 43 ----
hadooplibs/hadoop-distcp-2/pom.xml | 43 ----
hadooplibs/hadoop-distcp-3/pom.xml | 43 ----
hadooplibs/hadoop-utils-0.23/pom.xml | 42 ----
.../action/hadoop/LauncherMainHadoopUtils.java | 35 ----
.../apache/oozie/hadoop/utils/HadoopShims.java | 51 -----
hadooplibs/hadoop-utils-1/pom.xml | 41 ----
.../action/hadoop/LauncherMainHadoopUtils.java | 35 ----
.../apache/oozie/hadoop/utils/HadoopShims.java | 51 -----
hadooplibs/hadoop-utils-2/pom.xml | 42 ----
.../action/hadoop/LauncherMainHadoopUtils.java | 118 -----------
.../apache/oozie/hadoop/utils/HadoopShims.java | 68 -------
hadooplibs/hadoop-utils-3/pom.xml | 42 ----
.../action/hadoop/LauncherMainHadoopUtils.java | 115 -----------
.../apache/oozie/hadoop/utils/HadoopShims.java | 68 -------
hadooplibs/pom.xml | 93 ---------
pom.xml | 92 +--------
release-log.txt | 1 +
sharelib/distcp/pom.xml | 4 +-
.../apache/oozie/action/hadoop/DistcpMain.java | 2 +-
sharelib/hive/pom.xml | 12 +-
.../apache/oozie/action/hadoop/HiveMain.java | 2 +-
sharelib/hive2/pom.xml | 6 +-
.../apache/oozie/action/hadoop/Hive2Main.java | 2 +-
sharelib/oozie/pom.xml | 6 -
.../apache/oozie/action/hadoop/JavaMain.java | 2 +-
.../oozie/action/hadoop/LauncherMain.java | 92 ++++++++-
.../oozie/action/hadoop/LauncherMapper.java | 12 +-
.../oozie/action/hadoop/MapReduceMain.java | 2 +-
sharelib/pig/pom.xml | 10 +-
.../org/apache/oozie/action/hadoop/PigMain.java | 2 +-
sharelib/spark/pom.xml | 201 +++++++++----------
.../apache/oozie/action/hadoop/SparkMain.java | 2 +-
sharelib/sqoop/pom.xml | 5 -
.../apache/oozie/action/hadoop/SqoopMain.java | 2 +-
webapp/pom.xml | 8 +-
68 files changed, 381 insertions(+), 1561 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/client/pom.xml
----------------------------------------------------------------------
diff --git a/client/pom.xml b/client/pom.xml
index 632d256..a3c8238 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -74,8 +74,8 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/client/src/main/java/org/apache/oozie/client/XOozieClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/XOozieClient.java b/client/src/main/java/org/apache/oozie/client/XOozieClient.java
index 8ee9c57..60cedc9 100644
--- a/client/src/main/java/org/apache/oozie/client/XOozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/XOozieClient.java
@@ -33,19 +33,10 @@ import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
public class XOozieClient extends OozieClient {
-
- public static final String JT = "mapred.job.tracker";
- public static final String JT_2 = "mapreduce.jobtracker.address";
-
+ public static final String RM = "yarn.resourcemanager.address";
public static final String NN = "fs.default.name";
public static final String NN_2 = "fs.defaultFS";
- @Deprecated
- public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
-
- @Deprecated
- public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal";
-
public static final String PIG_SCRIPT = "oozie.pig.script";
public static final String PIG_OPTIONS = "oozie.pig.options";
@@ -123,12 +114,9 @@ public class XOozieClient extends OozieClient {
}
private void validateHttpSubmitConf(Properties conf) {
- String JT = conf.getProperty(XOozieClient.JT);
- String JT_2 = conf.getProperty(XOozieClient.JT_2);
- if (JT == null) {
- if(JT_2 == null) {
- throw new RuntimeException("jobtracker is not specified in conf");
- }
+ String RM = conf.getProperty(XOozieClient.RM);
+ if (RM == null) {
+ throw new RuntimeException("jobtracker is not specified in conf");
}
String NN = conf.getProperty(XOozieClient.NN);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index b72ea7d..6584af8 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -149,8 +149,8 @@
</dependency>
<dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 6893bb8..99e3344 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AccessControlException;
-import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -72,12 +71,12 @@ import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.util.ELEvaluationException;
import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.XLog;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.PropertiesUtils;
-import org.apache.oozie.util.XConfiguration;
-import org.apache.oozie.util.XLog;
-import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
@@ -86,8 +85,6 @@ import org.jdom.Namespace;
public class JavaActionExecutor extends ActionExecutor {
protected static final String HADOOP_USER = "user.name";
- public static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
- public static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address";
public static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
public static final String HADOOP_NAME_NODE = "fs.default.name";
private static final String HADOOP_JOB_NAME = "mapred.job.name";
@@ -127,9 +124,7 @@ public class JavaActionExecutor extends ActionExecutor {
static {
DISALLOWED_PROPERTIES.add(HADOOP_USER);
- DISALLOWED_PROPERTIES.add(HADOOP_JOB_TRACKER);
DISALLOWED_PROPERTIES.add(HADOOP_NAME_NODE);
- DISALLOWED_PROPERTIES.add(HADOOP_JOB_TRACKER_2);
DISALLOWED_PROPERTIES.add(HADOOP_YARN_RM);
}
@@ -145,8 +140,7 @@ public class JavaActionExecutor extends ActionExecutor {
List<Class> classes = new ArrayList<Class>();
classes.add(LauncherMapper.class);
classes.add(OozieLauncherInputFormat.class);
- classes.add(LauncherMainHadoopUtils.class);
- classes.add(HadoopShims.class);
+ classes.add(LauncherMain.class);
classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
return classes;
}
@@ -222,8 +216,6 @@ public class JavaActionExecutor extends ActionExecutor {
conf = new JobConf(false);
}
conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER));
- conf.set(HADOOP_JOB_TRACKER, jobTracker);
- conf.set(HADOOP_JOB_TRACKER_2, jobTracker);
conf.set(HADOOP_YARN_RM, jobTracker);
conf.set(HADOOP_NAME_NODE, nameNode);
conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "true");
@@ -255,7 +247,7 @@ public class JavaActionExecutor extends ActionExecutor {
XConfiguration launcherConf = new XConfiguration();
// Inject action defaults for launcher
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
- XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_JOB_TRACKER), getType());
+ XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_YARN_RM), getType());
injectLauncherProperties(actionDefaultConf, launcherConf);
// Inject <job-xml> and <configuration> for launcher
try {
@@ -524,7 +516,7 @@ public class JavaActionExecutor extends ActionExecutor {
throws ActionExecutorException {
try {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
- XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType());
+ XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_YARN_RM), getType());
XConfiguration.injectDefaults(actionDefaults, actionConf);
has.checkSupportedFilesystem(appPath.toUri());
@@ -1143,7 +1135,7 @@ public class JavaActionExecutor extends ActionExecutor {
if (alreadyRunning && !isUserRetry) {
runningJob = jobClient.getJob(JobID.forName(launcherId));
if (runningJob == null) {
- String jobTracker = launcherJobConf.get(HADOOP_JOB_TRACKER);
+ String jobTracker = launcherJobConf.get(HADOOP_YARN_RM);
throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA017",
"unknown job [{0}@{1}], cannot recover", launcherId, jobTracker);
}
@@ -1190,7 +1182,7 @@ public class JavaActionExecutor extends ActionExecutor {
LOG.debug("After submission get the launcherId " + launcherId);
}
- String jobTracker = launcherJobConf.get(HADOOP_JOB_TRACKER);
+ String jobTracker = launcherJobConf.get(HADOOP_YARN_RM);
String consoleUrl = runningJob.getTrackingURL();
context.setStartData(launcherId, jobTracker, consoleUrl);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
index ef6b99d..69e1044 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.oozie.client.OozieClient;
-import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
@@ -165,12 +164,12 @@ public class LauncherMapperHelper {
public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String launcherTag,
long launcherTime)
throws NoSuchAlgorithmException {
- launcherJobConf.setLong(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME, launcherTime);
+ launcherJobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, launcherTime);
// Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length)
String tag = getTag(launcherTag);
// keeping the oozie.child.mapreduce.job.tags instead of mapreduce.job.tags to avoid killing launcher itself.
// mapreduce.job.tags should only go to child job launch by launcher.
- actionConf.set(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS, tag);
+ actionConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, tag);
}
private static String getTag(String launcherTag) throws NoSuchAlgorithmException {
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 97355fd..252f461 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -40,8 +40,7 @@ import org.jdom.Namespace;
public class SparkActionExecutor extends JavaActionExecutor {
public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain";
- public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; // hadoop-2
- public static final String TASK_USER_CLASSPATH_PRECEDENCE = "mapreduce.user.classpath.first"; // hadoop-1
+ public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence";
public static final String SPARK_MASTER = "oozie.spark.master";
public static final String SPARK_MODE = "oozie.spark.mode";
public static final String SPARK_OPTS = "oozie.spark.spark-opts";
@@ -81,7 +80,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
StringBuilder sparkOptsSb = new StringBuilder();
if (master.startsWith("yarn")) {
- String resourceManager = actionConf.get(HADOOP_JOB_TRACKER);
+ String resourceManager = actionConf.get(HADOOP_YARN_RM);
Map<String, String> sparkConfig = Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager);
for (Map.Entry<String, String> entry : sparkConfig.entrySet()) {
sparkOptsSb.append("--conf ").append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
@@ -106,9 +105,6 @@ public class SparkActionExecutor extends JavaActionExecutor {
if (launcherJobConf.get("oozie.launcher." + TASK_USER_PRECEDENCE) == null) {
launcherJobConf.set(TASK_USER_PRECEDENCE, "true");
}
- if (launcherJobConf.get("oozie.launcher." + TASK_USER_CLASSPATH_PRECEDENCE) == null) {
- launcherJobConf.set(TASK_USER_CLASSPATH_PRECEDENCE, "true");
- }
return launcherJobConf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
index d2a2742..331bcf8 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
@@ -58,7 +58,7 @@ public abstract class SubmitHttpXCommand extends WorkflowXCommand<String> {
protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
static {
- MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
+ MANDATORY_OOZIE_CONFS.add(XOozieClient.RM);
MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
MANDATORY_OOZIE_CONFS.add(OozieClient.LIBPATH);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
index 9124a45..64fa89a 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
@@ -41,14 +41,15 @@ public class SubmitMRXCommand extends SubmitHttpXCommand {
static {
SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER);
- SKIPPED_CONFS.add(XOozieClient.JT);
+ SKIPPED_CONFS.add(XOozieClient.RM);
SKIPPED_CONFS.add(XOozieClient.NN);
// a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS'
// in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1
SKIPPED_CONFS.add(XOozieClient.NN_2);
DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2);
- DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2);
+ DEPRECATE_MAP.put(XOozieClient.RM, "yarn.resourcemanager.address");
+ DEPRECATE_MAP.put(XOozieClient.NN, "fs.defaultFS");
DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER, "mapreduce.job.user.name");
}
@@ -96,8 +97,8 @@ public class SubmitMRXCommand extends SubmitHttpXCommand {
protected Element generateSection(Configuration conf, Namespace ns) {
Element mapreduce = new Element("map-reduce", ns);
Element jt = new Element("job-tracker", ns);
- String newJTVal = conf.get(DEPRECATE_MAP.get(XOozieClient.JT));
- jt.addContent(newJTVal != null ? newJTVal : (conf.get(XOozieClient.JT)));
+ String newJTVal = conf.get(DEPRECATE_MAP.get(XOozieClient.RM));
+ jt.addContent(newJTVal != null ? newJTVal : (conf.get(XOozieClient.RM)));
mapreduce.addContent(jt);
Element nn = new Element("name-node", ns);
String newNNVal = conf.get(DEPRECATE_MAP.get(XOozieClient.NN));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
index 9d41305..b82369c 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
@@ -50,7 +50,7 @@ public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand {
String name = getWorkflowName();
Element ele = new Element(name, ns);
Element jt = new Element("job-tracker", ns);
- jt.addContent(conf.get(XOozieClient.JT));
+ jt.addContent(conf.get(XOozieClient.RM));
ele.addContent(jt);
Element nn = new Element("name-node", ns);
nn.addContent(conf.get(XOozieClient.NN));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
index 51b739e..51f3d03 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
@@ -50,7 +50,7 @@ public class SubmitSqoopXCommand extends SubmitHttpXCommand {
String name = "sqoop";
Element ele = new Element(name, ns);
Element jt = new Element("job-tracker", ns);
- jt.addContent(conf.get(XOozieClient.JT));
+ jt.addContent(conf.get(XOozieClient.RM));
ele.addContent(jt);
Element nn = new Element("name-node", ns);
nn.addContent(conf.get(XOozieClient.NN));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index f171ea3..794e825 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -82,8 +82,6 @@ public class HadoopAccessorService implements Service {
protected static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
/** The Kerberos principal for the resource manager.*/
protected static final String RM_PRINCIPAL = "yarn.resourcemanager.principal";
- protected static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
- protected static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address";
protected static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
private static final Map<String, Text> mrTokenRenewers = new HashMap<String, Text>();
@@ -491,7 +489,7 @@ public class HadoopAccessorService implements Service {
if (!conf.getBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, false)) {
throw new HadoopAccessorException(ErrorCode.E0903);
}
- String jobTracker = conf.get(JavaActionExecutor.HADOOP_JOB_TRACKER);
+ String jobTracker = conf.get(JavaActionExecutor.HADOOP_YARN_RM);
validateJobTracker(jobTracker);
try {
UserGroupInformation ugi = getUGI(user);
@@ -606,10 +604,7 @@ public class HadoopAccessorService implements Service {
renewer = mrTokenRenewers.get(servicePrincipal);
if (renewer == null) {
// Mimic org.apache.hadoop.mapred.Master.getMasterPrincipal()
- String target = jobConf.get(HADOOP_YARN_RM, jobConf.get(HADOOP_JOB_TRACKER_2));
- if (target == null) {
- target = jobConf.get(HADOOP_JOB_TRACKER);
- }
+ String target = jobConf.get(HADOOP_YARN_RM);
try {
String addr = NetUtils.createSocketAddr(target).getHostName();
renewer = new Text(SecurityUtil.getServerPrincipal(servicePrincipal, addr));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 66fd864..fa230da 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -51,14 +51,13 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.client.rest.JsonUtils;
-import org.apache.oozie.hadoop.utils.HadoopShims;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.oozie.ErrorCode;
import org.apache.oozie.util.Instrumentable;
import org.apache.oozie.util.Instrumentation;
+import org.apache.oozie.util.FSUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.oozie.ErrorCode;
import org.jdom.JDOMException;
public class ShareLibService implements Service, Instrumentable {
@@ -217,7 +216,7 @@ public class ShareLibService implements Service, Instrumentable {
*
* @param fs the FileSystem
* @param path the Path
- * @param perm is permission
+ * @param fsPerm is permission
* @throws IOException Signals that an I/O exception has occurred.
*/
private void recursiveChangePermissions(FileSystem fs, Path path, FsPermission fsPerm) throws IOException {
@@ -352,14 +351,12 @@ public class ShareLibService implements Service, Instrumentable {
}
private void checkSymlink(String shareLibKey) throws IOException {
- if (!HadoopShims.isSymlinkSupported() || symlinkMapping.get(shareLibKey) == null
- || symlinkMapping.get(shareLibKey).isEmpty()) {
+ if (symlinkMapping.get(shareLibKey) == null || symlinkMapping.get(shareLibKey).isEmpty()) {
return;
}
- HadoopShims fileSystem = new HadoopShims(fs);
for (Path path : symlinkMapping.get(shareLibKey).keySet()) {
- if (!symlinkMapping.get(shareLibKey).get(path).equals(fileSystem.getSymLinkTarget(path))) {
+ if (!symlinkMapping.get(shareLibKey).get(path).equals(FSUtils.getSymLinkTarget(fs, path))) {
synchronized (ShareLibService.class) {
Map<String, List<Path>> tmpShareLibMap = new HashMap<String, List<Path>>(shareLibMap);
@@ -370,7 +367,7 @@ public class ShareLibService implements Service, Instrumentable {
symlinkMapping);
LOG.info(MessageFormat.format("Symlink target for [{0}] has changed, was [{1}], now [{2}]",
- shareLibKey, path, fileSystem.getSymLinkTarget(path)));
+ shareLibKey, path, FSUtils.getSymLinkTarget(fs, path)));
loadShareLibMetaFile(tmpShareLibMap, tmpSymlinkMapping, tmpShareLibConfigMap, sharelibMappingFile,
shareLibKey);
shareLibMap = tmpShareLibMap;
@@ -636,19 +633,18 @@ public class ShareLibService implements Service, Instrumentable {
throws IOException {
List<Path> listOfPaths = new ArrayList<Path>();
Map<Path, Path> symlinkMappingforAction = new HashMap<Path, Path>();
- HadoopShims fileSystem = new HadoopShims(fs);
for (String dfsPath : pathList) {
Path path = new Path(dfsPath);
getPathRecursively(fs, new Path(dfsPath), listOfPaths, shareLibKey, shareLibConfigMap);
- if (HadoopShims.isSymlinkSupported() && fileSystem.isSymlink(path)) {
- symlinkMappingforAction.put(path, fileSystem.getSymLinkTarget(path));
+ if (FSUtils.isSymlink(fs, path)) {
+ symlinkMappingforAction.put(path, FSUtils.getSymLinkTarget(fs, path));
}
}
- if (HadoopShims.isSymlinkSupported()) {
- LOG.info("symlink for " + shareLibKey + ":" + symlinkMappingforAction);
- tmpSymlinkMapping.put(shareLibKey, symlinkMappingforAction);
- }
+
+ LOG.info("symlink for " + shareLibKey + ":" + symlinkMappingforAction);
+ tmpSymlinkMapping.put(shareLibKey, symlinkMappingforAction);
+
tmpShareLibMap.put(shareLibKey, listOfPaths);
LOG.info("Share lib for " + shareLibKey + ":" + listOfPaths);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/FSUtils.java b/core/src/main/java/org/apache/oozie/util/FSUtils.java
new file mode 100644
index 0000000..6d73fc7
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/util/FSUtils.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.util;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.net.URI;
+
+public final class FSUtils {
+ public static Path getSymLinkTarget(FileSystem fs, Path p) throws IOException {
+ try {
+ //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink
+ Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
+ return fs.getFileLinkStatus(tempPath).getSymlink();
+ }
+ catch (java.net.URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public static boolean isSymlink(FileSystem fs, Path p) throws IOException {
+ try {
+ //isSymlink doesn't work with fragment name, need to remove fragment before checking for symlink
+ Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath());
+ return fs.getFileLinkStatus(tempPath).isSymlink();
+ }
+ catch (java.net.URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public static void createSymlink(FileSystem fs, Path target, Path link, boolean createParent) throws IOException {
+ fs.createSymlink(target, link, createParent);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/main/java/org/apache/oozie/util/JobUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java
index a3a5fc0..cb53bc7 100644
--- a/core/src/main/java/org/apache/oozie/util/JobUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java
@@ -138,8 +138,7 @@ public class JobUtils {
}
/**
- * This method provides a wrapper around hadoop 0.20/1.x and 0.23/2.x implementations.
- * TODO: Remove the workaround when we drop the support for hadoop 0.20.
+ * This method provides a wrapper around hadoop 2.x implementations.
* @param file Path of the file to be added
* @param conf Configuration that contains the classpath setting
* @param fs FileSystem with respect to which path should be interpreted (may be null)
@@ -148,24 +147,12 @@ public class JobUtils {
public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException {
if (fs == null) {
Configuration defaultConf = Services.get().get(HadoopAccessorService.class)
- .createJobConf(conf.get(JavaActionExecutor.HADOOP_JOB_TRACKER));
+ .createJobConf(conf.get(JavaActionExecutor.HADOOP_YARN_RM));
XConfiguration.copy(conf, defaultConf);
// it fails with conf, therefore we pass defaultConf instead
fs = file.getFileSystem(defaultConf);
}
- // Hadoop 0.20/1.x.
- if (Services.get().get(HadoopAccessorService.class).getCachedConf().get("yarn.resourcemanager.webapp.address") == null) {
- // Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20
- // Refer OOZIE-1806.
- String filepath = file.toUri().getPath();
- String classpath = conf.get("mapred.job.classpath.files");
- conf.set("mapred.job.classpath.files",
- classpath == null ? filepath : classpath + System.getProperty("path.separator") + filepath);
- URI uri = fs.makeQualified(file).toUri();
- DistributedCache.addCacheFile(uri, conf);
- }
- else { // Hadoop 0.23/2.x
- DistributedCache.addFileToClassPath(file, conf, fs);
- }
+
+ DistributedCache.addFileToClassPath(file, conf, fs);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
index 5345ae6..7f01680 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
@@ -538,9 +538,7 @@ public class TestFsActionExecutor extends ActionExecutorTestCase {
assertEquals("rwx------", fs.getFileStatus(new Path(basePath + "/10/dir1")).getPermission().toString());
assertEquals("rwx------", fs.getFileStatus(new Path(basePath + "/10/dir2")).getPermission().toString());
assertEquals("rwx------", fs.getFileStatus(new Path(basePath + "/11/dir3")).getPermission().toString());
- // HDFS-4659 introduced an incompatible change that causes the following to be "rwx------" when run against Hadoop 2.1.x
- // but in Hadoop 1.x its still "rw-------" so we'll just skip verifying this for now.
- //assertEquals("rw-------", fs.getFileStatus(new Path(basePath + "/10/dir1/file1")).getPermission().toString());
+ assertEquals("rwx------", fs.getFileStatus(new Path(basePath + "/10/dir1/file1")).getPermission().toString());
fs.delete(basePath, true);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 057c9fb..879bfeb 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -54,7 +54,6 @@ import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
-import org.apache.oozie.hadoop.utils.HadoopShims;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
@@ -63,6 +62,7 @@ import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
+import org.apache.oozie.util.FSUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -113,7 +113,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
conf = new XConfiguration();
- conf.set("mapred.job.tracker", "a");
+ conf.set("yarn.resourcemanager.address", "a");
try {
JavaActionExecutor.checkForDisallowedProps(conf, "x");
fail();
@@ -200,7 +200,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createBaseHadoopConf(context, actionXml);
assertEquals(protoConf.get(WorkflowAppService.HADOOP_USER), conf.get(WorkflowAppService.HADOOP_USER));
- assertEquals(getJobTrackerUri(), conf.get("mapred.job.tracker"));
+ assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
assertEquals(getNameNodeUri(), conf.get("fs.default.name"));
conf = ae.createBaseHadoopConf(context, actionXml);
@@ -350,7 +350,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertNotNull(consoleUrl);
JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
- jobConf.set("mapred.job.tracker", jobTracker);
+ jobConf.set("yarn.resourcemanager.address", jobTracker);
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(getTestUser(), jobConf);
@@ -1991,12 +1991,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("-Xmx200m", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS));
assertEquals("-Xmx1024m -Djava.io.tmpdir=./usr", jobConf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS));
assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS));
- // There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1
- if (HadoopShims.isYARN()) {
- assertEquals("-Xmx1024m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- } else {
- assertNull(jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- }
+ assertEquals("-Xmx1024m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
}
public void testUpdateConfForUberMode() throws Exception {
Element actionXml1 = XmlUtils
@@ -2032,15 +2027,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
launcherConf.get("mapred.child.java.opts"));
assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
launcherConf.get("mapreduce.map.java.opts"));
- // There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1
- if (HadoopShims.isYARN()) {
- assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp",
- launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- }
- else {
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp",
- launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- }
+
+ assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp",
+ launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
+
assertEquals(2560, heapSize);
// env
@@ -2375,21 +2365,14 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// Test when server side setting is not enabled
JobConf launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
- if (HadoopShims.isYARN()) {
- assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
- } else {
- assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
- }
+ assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
ConfigurationService.set("oozie.action.launcher." + JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true");
// Test when server side setting is enabled but tez-site.xml is not in DistributedCache
launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
- if (HadoopShims.isYARN()) {
- assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
- } else {
- assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
- }
+ assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
+
final Path tezSite = new Path("/tmp/tez-site.xml");
final FSDataOutputStream out = getFileSystem().create(tezSite);
out.close();
@@ -2479,9 +2462,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf.set(WorkflowAppService.HADOOP_USER, getTestUser());
ae.addToCache(conf, appPath, appJarFullPath.toString(), false);
// assert that mapred.cache.files contains jar URI path (full on Hadoop-2)
- Path jarPath = HadoopShims.isYARN() ?
- new Path(appJarFullPath.toUri()):
- new Path(appJarFullPath.toUri().getPath());
+ Path jarPath = new Path(appJarFullPath.toUri());
assertTrue(conf.get("mapred.cache.files").contains(jarPath.toString()));
// assert that dist cache classpath contains jar URI path
Path[] paths = DistributedCache.getFileClassPaths(conf);
@@ -2778,25 +2759,25 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
Configuration conf = new Configuration(false);
Assert.assertEquals(0, conf.size());
- conf.set("mapred.job.tracker", getJobTrackerUri());
+ conf.set("yarn.resourcemanager.address", getJobTrackerUri());
ae.setupLauncherConf(conf, actionXmlWithConfiguration, null, null);
- assertEquals(getJobTrackerUri(), conf.get("mapred.job.tracker"));
+ assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
assertEquals("AA", conf.get("oozie.launcher.a"));
assertEquals("AA", conf.get("a"));
assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo"));
assertEquals("action.barbar", conf.get("action.foofoo"));
assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
if (conf.size() == 7) {
- assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address"));
+ assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
} else {
assertEquals(6, conf.size());
}
conf = new Configuration(false);
Assert.assertEquals(0, conf.size());
- conf.set("mapred.job.tracker", getJobTrackerUri());
+ conf.set("yarn.resourcemanager.address", getJobTrackerUri());
ae.setupLauncherConf(conf, actionXmlWithoutConfiguration, null, null);
- assertEquals(getJobTrackerUri(), conf.get("mapred.job.tracker"));
+ assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo"));
assertEquals("action.barbar", conf.get("action.foofoo"));
assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
index 54bfc16..b939f58 100644
--- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
+++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
@@ -31,7 +31,6 @@ import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.oozie.BaseEngine;
import org.apache.oozie.BuildInfo;
import org.apache.oozie.cli.CLIParser;
import org.apache.oozie.cli.OozieCLI;
@@ -136,7 +135,7 @@ public class TestOozieCLI extends DagServletTestCase {
Properties props = new Properties();
props.setProperty(OozieClient.USER_NAME, getTestUser());
props.setProperty(XOozieClient.NN, "localhost:9000");
- props.setProperty(XOozieClient.JT, "localhost:9001");
+ props.setProperty(XOozieClient.RM, "localhost:9001");
props.setProperty("oozie.libpath", appPath);
props.setProperty("mapred.output.dir", appPath);
props.setProperty("a", "A");
@@ -155,7 +154,7 @@ public class TestOozieCLI extends DagServletTestCase {
props.setProperty(OozieClient.APP_PATH, appPath);
props.setProperty(OozieClient.RERUN_SKIP_NODES, "node");
props.setProperty(XOozieClient.NN, "localhost:9000");
- props.setProperty(XOozieClient.JT, "localhost:9001");
+ props.setProperty(XOozieClient.RM, "localhost:9001");
if (useNewAPI) {
props.setProperty("mapreduce.map.class", "mapper.class");
props.setProperty("mapreduce.reduce.class", "reducer.class");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
index b06f81d..26d946f 100644
--- a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
+++ b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
@@ -29,8 +29,6 @@ import org.apache.oozie.servlet.MockDagEngineService;
import org.apache.oozie.servlet.V1JobsServlet;
import org.apache.oozie.servlet.V1AdminServlet;
-import java.io.File;
-
public class TestWorkflowXClient extends DagServletTestCase {
static {
@@ -60,7 +58,7 @@ public class TestWorkflowXClient extends DagServletTestCase {
Path libPath = new Path(getFsTestCaseDir(), "lib");
getFileSystem().mkdirs(libPath);
conf.setProperty(OozieClient.LIBPATH, libPath.toString());
- conf.setProperty(XOozieClient.JT, "localhost:9001");
+ conf.setProperty(XOozieClient.RM, "localhost:9001");
conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
String[] params = new String[]{"INPUT=input.txt"};
@@ -90,7 +88,7 @@ public class TestWorkflowXClient extends DagServletTestCase {
getFileSystem().mkdirs(libPath);
System.out.println(libPath.toString());
conf.setProperty(OozieClient.LIBPATH, libPath.toString());
- conf.setProperty(XOozieClient.JT, "localhost:9001");
+ conf.setProperty(XOozieClient.RM, "localhost:9001");
conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
String[] params = new String[]{"NAME=test"};
@@ -120,7 +118,7 @@ public class TestWorkflowXClient extends DagServletTestCase {
getFileSystem().mkdirs(libPath);
System.out.println(libPath.toString());
conf.setProperty(OozieClient.LIBPATH, libPath.toString());
- conf.setProperty(XOozieClient.JT, "localhost:9001");
+ conf.setProperty(XOozieClient.RM, "localhost:9001");
conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END,
@@ -156,7 +154,7 @@ public class TestWorkflowXClient extends DagServletTestCase {
catch (RuntimeException exception) {
assertEquals("java.lang.RuntimeException: jobtracker is not specified in conf", exception.toString());
}
- conf.setProperty(XOozieClient.JT, "localhost:9001");
+ conf.setProperty(XOozieClient.RM, "localhost:9001");
try {
wc.submitMapReduce(conf);
fail("submit client without NN should throw exception");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
index 43edf5e..014daff 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
@@ -45,7 +45,7 @@ public class TestSubmitHiveXCommand extends XFsTestCase {
public void testWFXmlGeneration() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
conf.set(OozieClient.LIBPATH, "libpath");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
index 5bc5747..388ff94 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
@@ -51,7 +51,7 @@ public class TestSubmitMRXCommand extends XFsTestCase {
public void testWFXmlGeneration() throws Exception {
Configuration conf = new Configuration(false);
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
conf.set(OozieClient.LIBPATH, "libpath");
@@ -97,7 +97,7 @@ public class TestSubmitMRXCommand extends XFsTestCase {
public void testWFXmlGenerationNegative1() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
// conf.set(XOozieClient.LIBPATH, "libpath");
@@ -118,8 +118,8 @@ public class TestSubmitMRXCommand extends XFsTestCase {
public void testWFXmlGenerationNewConfigProps() throws Exception {
try {
Configuration conf = new Configuration(false);
- conf.set(XOozieClient.NN_2, "new_NN");
- conf.set(XOozieClient.JT_2, "new_JT");
+ conf.set(XOozieClient.NN, "new_NN");
+ conf.set(XOozieClient.RM, "new_JT");
conf.set("mapred.mapper.class", "TestMapper");
conf.set("mapred.reducer.class", "TestReducer");
conf.set("mapred.input.dir", "testInput");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
index 5a1de25..e3ce56b 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
@@ -46,7 +46,7 @@ public class TestSubmitPigXCommand extends XFsTestCase {
public void testWFXmlGeneration1() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
conf.set(OozieClient.LIBPATH, "libpath");
@@ -118,7 +118,7 @@ public class TestSubmitPigXCommand extends XFsTestCase {
public void testWFXmlGeneration2() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
conf.set(OozieClient.LIBPATH, "libpath");
@@ -169,7 +169,7 @@ public class TestSubmitPigXCommand extends XFsTestCase {
public void testWFXmlGenerationNegative1() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
// conf.set(XOozieClient.LIBPATH, "libpath");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
index 49b5028..05fef6a 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
@@ -46,7 +46,7 @@ public class TestSubmitSqoopXCommand extends XFsTestCase {
public void testWFXmlGeneration() throws Exception {
Configuration conf = new Configuration();
- conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.RM, "jobtracker");
conf.set(XOozieClient.NN, "namenode");
conf.set(OozieClient.LIBPATH, "libpath");
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
index bbe2594..96faa48 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
@@ -182,7 +182,7 @@ public class TestHadoopAccessorService extends XTestCase {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
JobConf jobConf = new JobConf(false);
assertEquals(new Text("oozie mr token"), has.getMRTokenRenewerInternal(jobConf));
- jobConf.set("mapred.job.tracker", "localhost:50300");
+ jobConf.set("yarn.resourcemanager.address", "localhost:50300");
jobConf.set("mapreduce.jobtracker.kerberos.principal", "mapred/_HOST@KDC.DOMAIN.COM");
assertEquals(new Text("mapred/localhost@KDC.DOMAIN.COM"), has.getMRTokenRenewerInternal(jobConf));
jobConf = new JobConf(false);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index 7de7d78..6f25452 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -43,8 +43,8 @@ import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.action.hadoop.PigActionExecutor;
import org.apache.oozie.action.hadoop.TestJavaActionExecutor;
import org.apache.oozie.client.OozieClient;
-import org.apache.oozie.hadoop.utils.HadoopShims;
import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.FSUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -495,11 +495,8 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
assertTrue(shareLibService.getShareLibJars("directjar").get(0).getName().endsWith("direct.jar"));
- // Skipping for hadoop - 1.x because symlink is not supported
- if (HadoopShims.isSymlinkSupported()) {
- assertTrue(
- shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("targetOfLinkFile.xml"));
- }
+ assertTrue(shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("targetOfLinkFile.xml"));
+
List<Path> listOfPaths = shareLibService.getShareLibJars("directjar");
for (Path p : listOfPaths) {
assertTrue(p.toString().startsWith("hdfs"));
@@ -615,11 +612,6 @@ public class TestShareLibService extends XFsTestCase {
@Test
public void testMetafileSymlink() throws ServiceException, IOException {
- // Assume.assumeTrue("Skipping for hadoop - 1.x",HadoopFileSystem.isSymlinkSupported());
- if (!HadoopShims.isSymlinkSupported()) {
- return;
- }
-
services = new Services();
setSystemProps();
Configuration conf = services.get(ConfigurationService.class).getConf();
@@ -651,15 +643,14 @@ public class TestShareLibService extends XFsTestCase {
createFile(hive_site.toString());
- HadoopShims fileSystem = new HadoopShims(fs);
- fileSystem.createSymlink(basePath, symlink, true);
- fileSystem.createSymlink(hive_site, symlink_hive_site, true);
+ FSUtils.createSymlink(fs, basePath, symlink, true);
+ FSUtils.createSymlink(fs, hive_site, symlink_hive_site, true);
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString());
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".hive_conf", "/user/test/" + symlink_hive_site.toString()
+ "#hive-site.xml");
createTestShareLibMetaFile(fs, prop);
- assertEquals(fileSystem.isSymlink(symlink), true);
+ assertEquals(FSUtils.isSymlink(fs, symlink), true);
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
@@ -667,9 +658,9 @@ public class TestShareLibService extends XFsTestCase {
ShareLibService shareLibService = Services.get().get(ShareLibService.class);
assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
assertEquals(shareLibService.getShareLibJars("hive_conf").size(), 1);
- new HadoopShims(fs).createSymlink(basePath1, symlink, true);
- new HadoopShims(fs).createSymlink(hive_site1, symlink_hive_site, true);
- assertEquals(new HadoopShims(fs).getSymLinkTarget(shareLibService.getShareLibJars("hive_conf").get(0)),
+ FSUtils.createSymlink(fs, basePath1, symlink, true);
+ FSUtils.createSymlink(fs, hive_site1, symlink_hive_site, true);
+ assertEquals(FSUtils.getSymLinkTarget(fs, shareLibService.getShareLibJars("hive_conf").get(0)),
hive_site1);
assertEquals(shareLibService.getShareLibJars("pig").size(), 3);
}
@@ -781,8 +772,7 @@ public class TestShareLibService extends XFsTestCase {
String symlinkTarget = linkDir.toString() + Path.SEPARATOR + "targetOfLinkFile.xml";
createFile(directJarPath);
createFile(symlinkTarget);
- HadoopShims fsShim = new HadoopShims(fs);
- fsShim.createSymlink(new Path(symlinkTarget), new Path(symlink), true);
+ FSUtils.createSymlink(fs, new Path(symlinkTarget), new Path(symlink), true);
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString());
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new", "/user/test/" + somethingNew.toString());
@@ -991,16 +981,11 @@ public class TestShareLibService extends XFsTestCase {
private void verifyFilesInDistributedCache(URI[] cacheFiles, String... files) {
String cacheFilesStr = Arrays.toString(cacheFiles);
- if (new HadoopShims(getFileSystem()).isYARN()) {
- // Hadoop 2 has two extra jars
- assertEquals(cacheFiles.length, files.length + 2);
- assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
- assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
+ // Hadoop 2 has the following jars too: MRAppJar.jar and hadoop-mapreduce-client-jobclient-
+ assertEquals(cacheFiles.length, files.length + 2);
+ assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
+ assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
- }
- else {
- assertEquals(cacheFiles.length, files.length);
- }
for (String file : files) {
assertTrue(cacheFilesStr.contains(file));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
index 9002b6c..ef6d019 100644
--- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
+++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
@@ -61,7 +61,6 @@ public class TestLiteWorkflowAppParser extends XTestCase {
private String cleanupXml(String xml) {
xml = xml.replaceAll(" xmlns=?(\"|\')(\"|\')", "");
xml = xml.replaceAll("\\s*<source>.*</source>", ""); // remove the <source> added by Hadoop 2
- xml = xml.replaceAll("\\s*<!--Loaded from Unknown-->", ""); // remove the <!--LoadedfromUnknown--> added by Hadoop 1.2.1
return xml;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/distro/src/main/bin/addtowar.sh
----------------------------------------------------------------------
diff --git a/distro/src/main/bin/addtowar.sh b/distro/src/main/bin/addtowar.sh
index 3990409..9f8d53d 100644
--- a/distro/src/main/bin/addtowar.sh
+++ b/distro/src/main/bin/addtowar.sh
@@ -95,29 +95,13 @@ function checkOption() {
#get the list of hadoop jars that will be injected based on the hadoop version
function getHadoopJars() {
version=$1
- if [ "${version}" = "0.20.1" ]; then
- #List is separated by ":"
- hadoopJars="hadoop-core*.jar"
- elif [ "${version}" = "0.20.2" ]; then
- #List is separated by ":"
- hadoopJars="hadoop-core*.jar"
- elif [ "${version}" = "0.20.104" ]; then
- #List is separated by ":"
- hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar"
- elif [ "${version}" = "0.20.200" ]; then
- #List is separated by ":"
- hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar"
- elif [[ "${version}" =~ .*23 ]]; then
- suffix="-[0-9.]*"
- #List is separated by ":"
- hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:avro-ipc-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar"
- elif [[ "${version}" =~ 2.* ]]; then
+ if [[ "${version}" =~ 2.* ]]; then
suffix="-[0-9.]*"
#List is separated by ":"
hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-yarn-client${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-io-*.jar"
else
echo
- echo "Exiting: Unsupported Hadoop version '${hadoopVer}', supported versions: 0.20.1, 0.20.2, 0.20.104, 0.20.200, 0.23.x and 2.x"
+ echo "Exiting: Unsupported Hadoop version '${hadoopVer}', supported versions: 2.x"
echo
cleanUp
exit -1;
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index 41ddbd8..9c7e3d7 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -41,17 +41,14 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe
2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are
available to customise the versions of the dependencies:
--P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop
-profile depending on the hadoop version used.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT
- for hadoop-3
+-Dhadoop.version=<version> - default 2.4.0
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.12.1
--Dpig.classifier=<classifier> - default none
+-Dpig.classifier=<classifier> - default h2
-Dsqoop.version=<version> - default 1.4.3
--Dsqoop.classifier=<classifier> - default hadoop100
--Dtomcat.version=<version> - default 6.0.41
+-Dsqoop.classifier=<classifier> - default hadoop200
+-Dtomcat.version=<version> - default 6.0.44
-Dopenjpa.version=<version> - default 2.2.2
-Dxerces.version=<version> - default 2.10.0
-Dcurator.version=<version> - default 2.5.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index e76368e..cce219e 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -112,9 +112,9 @@ Except for the options marked with =(*)=, the options can be specified in the =t
of the Oozie project. The options marked with =(*)= are used in Maven POMs, thus they don't take effect if
specified in the =test.properties= file (which is loaded by the =XTestCase= class at class initialization time).
-*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-1 or Hadoop-2) you wish to build Oozie against specifically. It will
-substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. Default version is 1.2.1
-for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.3.0*.
+*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-2) you wish to build Oozie against specifically. It will
+substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven.
+For Hadoop 2.x, the default (and minimum) version is 2.4.0.
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
@@ -212,10 +212,7 @@ $ bin/mkdistro.sh [-DskipTests]
Running =mkdistro.sh= will create the binary distribution of Oozie. The following options are available to customise
the versions of the dependencies:
-Puber - Bundle required hadoop and hcatalog libraries in oozie war
--P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop
-profile depending on the hadoop version used.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0
-.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 2.4.0
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.12.1
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-auth-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-auth-0.23/pom.xml b/hadooplibs/hadoop-auth-0.23/pom.xml
deleted file mode 100644
index 7b3b466..0000000
--- a/hadooplibs/hadoop-auth-0.23/pom.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Auth</description>
- <name>Apache Oozie Hadoop Auth ${project.version} Test</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-auth-1/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-auth-1/pom.xml b/hadooplibs/hadoop-auth-1/pom.xml
deleted file mode 100644
index 9da6c08..0000000
--- a/hadooplibs/hadoop-auth-1/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
- <version>hadoop-1-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Auth</description>
- <name>Apache Oozie Hadoop Auth ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-auth-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-auth-2/pom.xml b/hadooplibs/hadoop-auth-2/pom.xml
deleted file mode 100644
index 07df37b..0000000
--- a/hadooplibs/hadoop-auth-2/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
- <version>hadoop-2-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop</description>
- <name>Apache Oozie Hadoop Auth ${project.version} Test</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-distcp-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-distcp-0.23/pom.xml b/hadooplibs/hadoop-distcp-0.23/pom.xml
deleted file mode 100644
index 60c8e02..0000000
--- a/hadooplibs/hadoop-distcp-0.23/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Distcp ${project.version}</description>
- <name>Apache Oozie Hadoop Distcp ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-distcp-1/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-distcp-1/pom.xml b/hadooplibs/hadoop-distcp-1/pom.xml
deleted file mode 100644
index f617b69..0000000
--- a/hadooplibs/hadoop-distcp-1/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>hadoop-1-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Distcp ${project.version}</description>
- <name>Apache Oozie Hadoop Distcp ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-tools</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-distcp-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-distcp-2/pom.xml b/hadooplibs/hadoop-distcp-2/pom.xml
deleted file mode 100644
index b7c3c63..0000000
--- a/hadooplibs/hadoop-distcp-2/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>hadoop-2-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Distcp ${project.version}</description>
- <name>Apache Oozie Hadoop Distcp ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-distcp-3/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-distcp-3/pom.xml b/hadooplibs/hadoop-distcp-3/pom.xml
deleted file mode 100644
index 1bb2895..0000000
--- a/hadooplibs/hadoop-distcp-3/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-distcp</artifactId>
- <version>hadoop-3-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Distcp ${project.version}</description>
- <name>Apache Oozie Hadoop Distcp ${project.version}</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- </dependency>
- </dependencies>
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/pom.xml b/hadooplibs/hadoop-utils-0.23/pom.xml
deleted file mode 100644
index a55e647..0000000
--- a/hadooplibs/hadoop-utils-0.23/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-main</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-utils</artifactId>
- <version>hadoop-0.23-4.3.0-SNAPSHOT</version>
- <description>Apache Oozie Hadoop Utils</description>
- <name>Apache Oozie Hadoop Utils</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>