You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2015/01/10 18:39:46 UTC
[2/2] falcon git commit: FALCON-896 Upgrade oozie to 4.1.0 and make
it the default for falcon. Contributed by Peeyush Bishnoi
FALCON-896 Upgrade oozie to 4.1.0 and make it the default for falcon. Contributed by Peeyush Bishnoi
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/d2fd49e5
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/d2fd49e5
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/d2fd49e5
Branch: refs/heads/master
Commit: d2fd49e5c4aeae7046afef1a8d23fa696dd9a4eb
Parents: 4a9151e
Author: Srikanth Sundarrajan <sr...@apache.org>
Authored: Sat Jan 10 23:09:30 2015 +0530
Committer: Srikanth Sundarrajan <sr...@apache.org>
Committed: Sat Jan 10 23:09:30 2015 +0530
----------------------------------------------------------------------
Installation-steps.txt | 2 +-
build-tools/src/bin/build-oozie.sh | 9 +-
hadoop-webapp/pom.xml | 265 ------------------
.../org/apache/falcon/JobTrackerService.java | 28 --
.../falcon/listener/HadoopStartupListener.java | 90 -------
hadoop-webapp/src/main/resources/core-site.xml | 55 ----
hadoop-webapp/src/main/resources/hdfs-site.xml | 64 -----
hadoop-webapp/src/main/resources/hive-site.xml | 42 ---
hadoop-webapp/src/main/resources/log4j.xml | 38 ---
.../src/main/resources/mapred-site.xml | 68 -----
hadoop-webapp/src/main/resources/yarn-site.xml | 64 -----
hadoop-webapp/src/main/webapp/WEB-INF/web.xml | 32 ---
.../apache/hadoop/mapred/LocalRunnerTest.java | 37 ---
.../org/apache/hadoop/mapred/LocalRunnerV2.java | 242 -----------------
.../workflow/engine/OozieWorkflowEngine.java | 4 +-
pom.xml | 6 +-
test-tools/hadoop-webapp/pom.xml | 266 +++++++++++++++++++
.../org/apache/falcon/JobTrackerService.java | 28 ++
.../falcon/listener/HadoopStartupListener.java | 135 ++++++++++
.../src/main/resources/core-site.xml | 55 ++++
.../src/main/resources/hdfs-site.xml | 64 +++++
.../src/main/resources/hive-site.xml | 60 +++++
.../hadoop-webapp/src/main/resources/log4j.xml | 38 +++
.../src/main/resources/mapred-site.xml | 68 +++++
.../src/main/resources/yarn-site.xml | 64 +++++
.../src/main/webapp/WEB-INF/web.xml | 32 +++
.../apache/hadoop/mapred/LocalRunnerTest.java | 37 +++
.../org/apache/hadoop/mapred/LocalRunnerV2.java | 242 +++++++++++++++++
test-tools/hcatalog-sharelib/pom.xml | 64 +++++
test-tools/hive-sharelib/pom.xml | 64 +++++
test-tools/oozie-sharelib/pom.xml | 63 +++++
test-tools/pig-sharelib/pom.xml | 63 +++++
test-tools/pom.xml | 42 +++
webapp/pom.xml | 10 +-
.../conf/oozie/conf/hadoop-conf/core-site.xml | 5 +
.../conf/oozie/conf/hadoop-conf/mapred-site.xml | 28 ++
webapp/src/conf/oozie/conf/oozie-site.xml | 4 +-
37 files changed, 1433 insertions(+), 1045 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/Installation-steps.txt
----------------------------------------------------------------------
diff --git a/Installation-steps.txt b/Installation-steps.txt
index 2e15805..2cd1f11 100644
--- a/Installation-steps.txt
+++ b/Installation-steps.txt
@@ -125,4 +125,4 @@ d. Stopping Falcon Server
* cd <<project home>>
* src/bin/pacakge.sh <<hadoop-version>>
>> ex. src/bin/pacakge.sh 2.5.0
- >> oozie bundle available in target/package/oozie-4.0.0/distro/target/oozie-4.0.0-distro.tar.gz
+ >> oozie bundle available in target/package/oozie-4.1.0/distro/target/oozie-4.1.0-distro.tar.gz
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/build-tools/src/bin/build-oozie.sh
----------------------------------------------------------------------
diff --git a/build-tools/src/bin/build-oozie.sh b/build-tools/src/bin/build-oozie.sh
index 5474761..b5f3179 100755
--- a/build-tools/src/bin/build-oozie.sh
+++ b/build-tools/src/bin/build-oozie.sh
@@ -55,17 +55,12 @@ sed -i.bak s/$VERSION\<\\/version\>/$BUILD_VERSION\<\\/version\>/g pom.xml */pom
patch -p0 < ../../build-tools/src/patches/oozie-site.patch
case $VERSION in
-4.0.0 )
- patch -p1 --verbose < ../../build-tools/src/patches/OOZIE-1551-4.0.patch
- patch -p0 < ../../build-tools/src/patches/OOZIE-1741.patch
- ;;
-4.0.1 )
- patch -p0 < ../../build-tools/src/patches/OOZIE-1741.patch
+4.1.0 )
;;
esac
rm `find . -name 'pom.xml.bak'`
-$MVN_CMD clean install -DskipTests
+$MVN_CMD clean install -DskipTests -Phadoop-2
popd
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/pom.xml b/hadoop-webapp/pom.xml
deleted file mode 100644
index 7a2f905..0000000
--- a/hadoop-webapp/pom.xml
+++ /dev/null
@@ -1,265 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.falcon</groupId>
- <artifactId>falcon-main</artifactId>
- <version>0.7-SNAPSHOT</version>
- </parent>
- <artifactId>falcon-hadoop-webapp</artifactId>
- <description>Apache Falcon Embedded Hadoop - Test Cluster</description>
- <name>Apache Falcon Embedded Hadoop - Test Cluster</name>
- <packaging>war</packaging>
-
- <profiles>
- <profile>
- <id>hadoop-2</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <classifier>tests</classifier>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- <scope>compile</scope>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.falcon</groupId>
- <artifactId>falcon-oozie-el-extension</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.falcon</groupId>
- <artifactId>falcon-hadoop-dependencies</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.falcon</groupId>
- <artifactId>falcon-test-util</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.activemq</groupId>
- <artifactId>activemq-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </dependency>
-
- <!-- Hive Metastore and WebHcat fails with out these dependencies -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </dependency>
-
- <!-- Oozie dependencies -->
- <dependency>
- <groupId>org.apache.hcatalog</groupId>
- <artifactId>webhcat-java-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>explode-war</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>unpack</goal>
- </goals>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-webapp</artifactId>
- <type>war</type>
- <outputDirectory>${project.build.directory}/oozie</outputDirectory>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- <execution>
- <id>add-jars</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/oozie/WEB-INF/lib</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.6</version>
- <executions>
- <execution>
- <configuration>
- <target>
- <jar destfile="target/oozie.war" basedir="target/oozie"/>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java b/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
deleted file mode 100644
index fa3a84a..0000000
--- a/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.falcon;
-
-/**
- * Job Tracker service interface.
- */
-public interface JobTrackerService {
-
- void start() throws Exception;
-
- void stop() throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java b/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
deleted file mode 100644
index a203290..0000000
--- a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.listener;
-
-import org.apache.activemq.broker.BrokerService;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-
-/**
- * Listener for bootstrapping embedded hadoop cluster for integration tests.
- */
-public class HadoopStartupListener implements ServletContextListener {
- private static final Logger LOG = LoggerFactory.getLogger(HadoopStartupListener.class);
- private BrokerService broker;
-
- @Override
- public void contextInitialized(ServletContextEvent sce) {
- try {
- startBroker();
- startHiveMetaStore();
-
- } catch (Exception e) {
- LOG.error("Unable to start daemons", e);
- throw new RuntimeException("Unable to start daemons", e);
- }
- }
-
- private void startBroker() throws Exception {
- broker = new BrokerService();
- broker.setUseJmx(false);
- broker.setDataDirectory("target/data");
- broker.addConnector("vm://localhost");
- broker.addConnector("tcp://0.0.0.0:61616");
- broker.start();
- }
-
- public static final String META_STORE_PORT = "49083";
- private void startHiveMetaStore() {
- try {
- new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- String[] args = new String[]{
- "-v",
- "-p", META_STORE_PORT,
- };
-
- HiveMetaStore.main(args);
- } catch (Throwable t) {
- throw new RuntimeException(t);
- }
- }
- }).start();
- } catch (Exception e) {
- throw new RuntimeException("Unable to start hive metastore server.", e);
- }
- }
-
- @Override
- public void contextDestroyed(ServletContextEvent sce) {
- try {
- if (broker != null) {
- broker.stop();
- }
- } catch(Exception e) {
- LOG.warn("Failed to stop activemq", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/core-site.xml b/hadoop-webapp/src/main/resources/core-site.xml
deleted file mode 100644
index ef1558e..0000000
--- a/hadoop-webapp/src/main/resources/core-site.xml
+++ /dev/null
@@ -1,55 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>fs.fsext.impl</name>
- <value>org.apache.falcon.hadoop.FileSystemExtension</value>
- </property>
-
- <property>
- <name>fs.jail.impl</name>
- <value>org.apache.falcon.hadoop.JailedFileSystem</value>
- </property>
-
- <property>
- <name>fs.defaultFS</name>
- <value>jail://global:00</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.${user.name}.groups</name>
- <value>*</value>
- <description>Allow the superuser oozie to impersonate any members of the group group1 and group2</description>
- </property>
-
- <property>
- <name>hadoop.proxyuser.${user.name}.hosts</name>
- <value>*</value>
- <description>The superuser can connect only from host1 and host2 to impersonate a user</description>
- </property>
-
- <property>
- <name>hadoop.tmp.dir</name>
- <value>${project.build.directory}/tmp-hadoop-${user.name}</value>
- <description>A base for other temporary directories.</description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/hdfs-site.xml b/hadoop-webapp/src/main/resources/hdfs-site.xml
deleted file mode 100644
index 0b929e6..0000000
--- a/hadoop-webapp/src/main/resources/hdfs-site.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- <property>
- <name>dfs.replication</name>
- <value>1</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:41110</value>
- <description>
- The datanode server address and port for data transfer.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:41175</value>
- <description>
- The datanode http server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:41120</value>
- <description>
- The datanode ipc server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
- <property>
- <name>dfs.http.address</name>
- <value>0.0.0.0:41170</value>
- <description>
- The address and the base port where the dfs namenode web ui will listen on.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/hive-site.xml b/hadoop-webapp/src/main/resources/hive-site.xml
deleted file mode 100644
index 6b39592..0000000
--- a/hadoop-webapp/src/main/resources/hive-site.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>hive.metastore.uris</name>
- <value>thrift://localhost:49083</value>
- </property>
-
- <property>
- <name>hive.metastore.local</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.defaultFS</name>
- <value>jail://global:00</value>
- </property>
-
- <!-- Forcing the creation of the db dir under target -->
- <property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:;databaseName=./target/metastore_db;create=true</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/log4j.xml b/hadoop-webapp/src/main/resources/log4j.xml
deleted file mode 100644
index d69e921..0000000
--- a/hadoop-webapp/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
- <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="File" value="${user.dir}/target/logs/jetty.log"/>
- <param name="Append" value="true"/>
- <param name="Threshold" value="debug"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
- </layout>
- </appender>
-
- <root>
- <priority value="info"/>
- <appender-ref ref="FILE"/>
- </root>
-
-</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/mapred-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/mapred-site.xml b/hadoop-webapp/src/main/resources/mapred-site.xml
deleted file mode 100644
index cf297de..0000000
--- a/hadoop-webapp/src/main/resources/mapred-site.xml
+++ /dev/null
@@ -1,68 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- <property>
- <name>mapreduce.jobtracker.address</name>
- <value>localhost:41021</value>
- </property>
-
- <property>
- <name>mapred.tasktracker.map.tasks.maximum</name>
- <value>2</value>
- <description>The maximum number of map tasks that will be run
- simultaneously by a task tracker.
- </description>
- </property>
-
- <property>
- <name>mapred.tasktracker.reduce.tasks.maximum</name>
- <value>1</value>
- <description>The maximum number of reduce tasks that will be run
- simultaneously by a task tracker.
- </description>
- </property>
-
- <property>
- <name>mapreduce.jobtracker.staging.root.dir</name>
- <value>/user</value>
- </property>
-
- <property>
- <name>mapred.job.tracker.http.address</name>
- <value>0.0.0.0:41130</value>
- <description>
- The job tracker http server address and port the server will listen on.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
- <property>
- <name>mapred.task.tracker.http.address</name>
- <value>0.0.0.0:41160</value>
- <description>
- The task tracker http server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/yarn-site.xml b/hadoop-webapp/src/main/resources/yarn-site.xml
deleted file mode 100644
index 658752b..0000000
--- a/hadoop-webapp/src/main/resources/yarn-site.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- <property>
- <description>The address of the applications manager interface in the RM.</description>
- <name>yarn.resourcemanager.address</name>
- <value>0.0.0.0:41021</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>unittests</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>0.0.0.0:41025</value>
- </property>
-
- <property>
- <description>The address of the RM web application.</description>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>0.0.0.0:41130</value>
- </property>
-
- <property>
- <description>The address of the scheduler interface.</description>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>0.0.0.0:41030</value>
- </property>
-
- <property>
- <description>Address where the localizer IPC is.</description>
- <name>yarn.nodemanager.localizer.address</name>
- <value>0.0.0.0:41040</value>
- </property>
-
- <property>
- <description>The address of the RM admin interface.</description>
- <name>yarn.resourcemanager.admin.address</name>
- <value>0.0.0.0:41140</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/webapp/WEB-INF/web.xml b/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index 8b186ad..0000000
--- a/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
- "http://java.sun.com/dtd/web-app_2_3.dtd">
-
-<web-app>
-
- <display-name>Apache Falcon Embedded Hadoop - Test Cluster</display-name>
- <description>Apache Falcon Embedded Hadoop - Test Cluster</description>
-
- <listener>
- <listener-class>org.apache.falcon.listener.HadoopStartupListener</listener-class>
- </listener>
-
-</web-app>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java b/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
deleted file mode 100644
index 193fab0..0000000
--- a/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.conf.Configuration;
-import org.testng.annotations.Test;
-
-/**
- * Test for LocalRunner.
- */
-@Test (enabled = false)
-public class LocalRunnerTest {
-
- @SuppressWarnings("unchecked")
- public void testLocalRunner() throws Exception {
- Configuration conf = new Configuration();
- conf.set("mapreduce.jobtracker.address", "localhost:41021");
- conf.set("mapreduce.framework.name", "unittests");
- JobClient client = new JobClient(new JobConf(conf));
- System.out.println(client.getSystemDir());
- }
-}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/hadoop-webapp/src/versioned-src/v2/java/org/apache/hadoop/mapred/LocalRunnerV2.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/versioned-src/v2/java/org/apache/hadoop/mapred/LocalRunnerV2.java b/hadoop-webapp/src/versioned-src/v2/java/org/apache/hadoop/mapred/LocalRunnerV2.java
deleted file mode 100644
index ccb1bd5..0000000
--- a/hadoop-webapp/src/versioned-src/v2/java/org/apache/hadoop/mapred/LocalRunnerV2.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.falcon.JobTrackerService;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.mapreduce.Cluster;
-import org.apache.hadoop.mapreduce.ClusterMetrics;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.JobStatus;
-import org.apache.hadoop.mapreduce.QueueAclsInfo;
-import org.apache.hadoop.mapreduce.QueueInfo;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskCompletionEvent;
-import org.apache.hadoop.mapreduce.TaskReport;
-import org.apache.hadoop.mapreduce.TaskTrackerInfo;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.LogParams;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.security.token.Token;
-
-import java.io.IOException;
-
-/**
- * Local Job Runner for Hadoop v2.
- * Please note that one of org.apache.hadoop.mapred.LocalRunnerV2 or
- * org.apache.hadoop.mapred.LocalRunnerV2 is active in the project depending
- * on the profile chosen.
- */
-public class LocalRunnerV2 implements ClientProtocol, JobTrackerService {
-
- private final ClientProtocol localProxy;
- private final Configuration conf;
- private Server server;
-
- public LocalRunnerV2() {
- try {
- conf = new Configuration();
- localProxy = new LocalJobRunner(conf);
- } catch (IOException e) {
- throw new RuntimeException("Unable to initialize localRunner");
- }
- }
-
- @Override
- public void start() throws Exception {
- server = new RPC.Builder(conf).setBindAddress("0.0.0.0").setPort(41021).setInstance(this).
- setProtocol(ClientProtocol.class).build();
- server.start();
- }
-
- public void stop() throws Exception {
- server.stop();
- }
-
- @Override
- public JobID getNewJobID() throws IOException, InterruptedException {
- return localProxy.getNewJobID();
- }
-
- @Override
- public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
- throws IOException, InterruptedException {
- return localProxy.submitJob(jobId, jobSubmitDir, ts);
- }
-
- @Override
- public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
- return localProxy.getClusterMetrics();
- }
-
- @Override
- public Cluster.JobTrackerStatus getJobTrackerStatus() throws IOException, InterruptedException {
- return localProxy.getJobTrackerStatus();
- }
-
- @Override
- public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
- return localProxy.getTaskTrackerExpiryInterval();
- }
-
- @Override
- public AccessControlList getQueueAdmins(String queueName) throws IOException {
- return localProxy.getQueueAdmins(queueName);
- }
-
- @Override
- public void killJob(JobID jobid) throws IOException, InterruptedException {
- localProxy.killJob(jobid);
- }
-
- @Override
- public void setJobPriority(JobID jobid, String priority) throws IOException, InterruptedException {
- localProxy.setJobPriority(jobid, priority);
- }
-
- @Override
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail) throws IOException, InterruptedException {
- return localProxy.killTask(taskId, shouldFail);
- }
-
- @Override
- public JobStatus getJobStatus(JobID jobid) throws IOException, InterruptedException {
- return localProxy.getJobStatus(jobid);
- }
-
- @Override
- public Counters getJobCounters(JobID jobid) throws IOException, InterruptedException {
- return localProxy.getJobCounters(jobid);
- }
-
- @Override
- public TaskReport[] getTaskReports(JobID jobid, TaskType type) throws IOException, InterruptedException {
- return localProxy.getTaskReports(jobid, type);
- }
-
- @Override
- public String getFilesystemName() throws IOException, InterruptedException {
- return localProxy.getFilesystemName();
- }
-
- @Override
- public JobStatus[] getAllJobs() throws IOException, InterruptedException {
- return localProxy.getAllJobs();
- }
-
- @Override
- public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid, int fromEventId, int maxEvents)
- throws IOException, InterruptedException {
- return localProxy.getTaskCompletionEvents(jobid, fromEventId, maxEvents);
- }
-
- @Override
- public String[] getTaskDiagnostics(TaskAttemptID taskId) throws IOException, InterruptedException {
- return localProxy.getTaskDiagnostics(taskId);
- }
-
- @Override
- public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException {
- return localProxy.getActiveTrackers();
- }
-
- @Override
- public TaskTrackerInfo[] getBlacklistedTrackers() throws IOException, InterruptedException {
- return localProxy.getBlacklistedTrackers();
- }
-
- @Override
- public String getSystemDir() throws IOException, InterruptedException {
- return localProxy.getSystemDir();
- }
-
- @Override
- public String getStagingAreaDir() throws IOException, InterruptedException {
- return localProxy.getStagingAreaDir();
- }
-
- @Override
- public String getJobHistoryDir() throws IOException, InterruptedException {
- return localProxy.getJobHistoryDir();
- }
-
- @Override
- public QueueInfo[] getQueues() throws IOException, InterruptedException {
- return localProxy.getQueues();
- }
-
- @Override
- public QueueInfo getQueue(String queueName) throws IOException, InterruptedException {
- return localProxy.getQueue(queueName);
- }
-
- @Override
- public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException {
- return localProxy.getQueueAclsForCurrentUser();
- }
-
- @Override
- public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
- return localProxy.getRootQueues();
- }
-
- @Override
- public QueueInfo[] getChildQueues(String queueName) throws IOException, InterruptedException {
- return localProxy.getChildQueues(queueName);
- }
-
- @Override
- public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) throws IOException, InterruptedException {
- return localProxy.getDelegationToken(renewer);
- }
-
- @Override
- public long renewDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException, InterruptedException {
- return localProxy.renewDelegationToken(token);
- }
-
- @Override
- public void cancelDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException, InterruptedException {
- localProxy.cancelDelegationToken(token);
- }
-
- @Override
- public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID)
- throws IOException, InterruptedException {
- return localProxy.getLogFileParams(jobID, taskAttemptID);
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
- return localProxy.getProtocolVersion(protocol, clientVersion);
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash)
- throws IOException {
- return localProxy.getProtocolSignature(protocol, clientVersion, clientMethodsHash);
- }
-}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index f1f96b1..ce292bd 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -809,7 +809,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
private void reRunCoordAction(String cluster, CoordinatorAction coordinatorAction) throws FalconException {
try {
OozieClient client = OozieClientFactory.get(cluster);
- client.reRunCoord(coordinatorAction.getJobId(), RestConstants.JOB_COORD_RERUN_ACTION,
+ client.reRunCoord(coordinatorAction.getJobId(), RestConstants.JOB_COORD_ACTION_RERUN,
Integer.toString(coordinatorAction.getActionNumber()), true, true);
assertCoordActionStatus(cluster, coordinatorAction.getId(),
org.apache.oozie.client.CoordinatorAction.Status.RUNNING,
@@ -869,7 +869,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
|| CoordinatorAction.Status.WAITING.toString().equals(status)
|| CoordinatorAction.Status.SUBMITTED.toString().equals(status)) {
return InstancesResult.WorkflowStatus.WAITING.name();
- } else if (CoordinatorAction.Status.DISCARDED.toString().equals(status)) {
+ } else if (CoordinatorAction.Status.IGNORED.toString().equals(status)) {
return InstancesResult.WorkflowStatus.KILLED.name();
} else if (CoordinatorAction.Status.TIMEDOUT.toString().equals(status)) {
return InstancesResult.WorkflowStatus.FAILED.name();
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e39f250..cd1a8fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,7 +103,7 @@
<include.prism>true</include.prism>
<slf4j.version>1.7.5</slf4j.version>
- <oozie.version>4.0.1</oozie.version>
+ <oozie.version>4.1.0</oozie.version>
<oozie.buildversion>${oozie.version}-falcon</oozie.buildversion>
<oozie.forcebuild>false</oozie.forcebuild>
<activemq.version>5.4.3</activemq.version>
@@ -343,7 +343,7 @@
<configuration>
<!--debug>true</debug -->
<xmlOutput>true</xmlOutput>
- <excludeFilterFile>${basedir}/../checkstyle/src/main/resources/falcon/findbugs-exclude.xml</excludeFilterFile>
+ <excludeFilterFile>checkstyle/src/main/resources/falcon/findbugs-exclude.xml</excludeFilterFile>
<failOnError>true</failOnError>
</configuration>
<executions>
@@ -377,7 +377,7 @@
<module>common</module>
<module>test-util</module>
<module>hadoop-dependencies</module>
- <module>hadoop-webapp</module>
+ <module>test-tools</module>
<module>messaging</module>
<module>oozie-el-extensions</module>
<module>oozie</module>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/pom.xml b/test-tools/hadoop-webapp/pom.xml
new file mode 100644
index 0000000..71df231
--- /dev/null
+++ b/test-tools/hadoop-webapp/pom.xml
@@ -0,0 +1,266 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-main</artifactId>
+ <version>0.7-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <artifactId>falcon-hadoop-webapp</artifactId>
+ <description>Apache Falcon Embedded Hadoop - Test Cluster</description>
+ <name>Apache Falcon Embedded Hadoop - Test Cluster</name>
+ <packaging>war</packaging>
+
+ <profiles>
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-oozie-el-extension</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-hadoop-dependencies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-test-util</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ </dependency>
+
+ <!-- Hive Metastore and WebHcat fails with out these dependencies -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+
+ <!-- Oozie dependencies -->
+ <dependency>
+ <groupId>org.apache.hcatalog</groupId>
+ <artifactId>webhcat-java-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>explode-war</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-webapp</artifactId>
+ <type>war</type>
+ <outputDirectory>${project.build.directory}/oozie</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-jars</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/oozie/WEB-INF/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.6</version>
+ <executions>
+ <execution>
+ <configuration>
+ <target>
+ <jar destfile="target/oozie.war" basedir="target/oozie"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java b/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
new file mode 100644
index 0000000..fa3a84a
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/JobTrackerService.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon;
+
+/**
+ * Job Tracker service interface.
+ */
+public interface JobTrackerService {
+
+ void start() throws Exception;
+
+ void stop() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java b/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
new file mode 100644
index 0000000..4298558
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.listener;
+
+import org.apache.activemq.broker.BrokerService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * Listener for bootstrapping embedded hadoop cluster for integration tests.
+ */
+public class HadoopStartupListener implements ServletContextListener {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopStartupListener.class);
+ private BrokerService broker;
+ private final String shareLibPath = "target/share/lib";
+ private static final String SHARE_LIB_PREFIX = "lib_";
+ private static final String USER = System.getProperty("user.name");
+
+ @Override
+ public void contextInitialized(ServletContextEvent sce) {
+ try {
+ copyShareLib();
+ startBroker();
+ startHiveMetaStore();
+ } catch (Exception e) {
+ LOG.error("Unable to start daemons", e);
+ throw new RuntimeException("Unable to start daemons", e);
+ }
+ }
+
+ private void copyShareLib() throws Exception {
+ String shareLibHDFSPath = getShareLibPath() + File.separator + SHARE_LIB_PREFIX
+ + getTimestampDirectory();
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(new Path(shareLibHDFSPath).toUri(), conf);
+ String[] actionDirectories = getLibActionDirectories();
+ for(String actionDirectory : actionDirectories) {
+ LOG.info("Copying Action Directory {0}", actionDirectory);
+ fs.copyFromLocalFile(new Path(shareLibPath, actionDirectory), new Path(shareLibHDFSPath));
+ }
+ }
+
+ private void startBroker() throws Exception {
+ broker = new BrokerService();
+ broker.setUseJmx(false);
+ broker.setDataDirectory("target/data");
+ broker.addConnector("vm://localhost");
+ broker.addConnector("tcp://0.0.0.0:61616");
+ broker.start();
+ }
+
+ public static final String META_STORE_PORT = "49083";
+ private void startHiveMetaStore() {
+ try {
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ String[] args = new String[]{
+ "-v",
+ "-p", META_STORE_PORT,
+ };
+
+ HiveMetaStore.main(args);
+ } catch (Throwable t) {
+ throw new RuntimeException(t);
+ }
+ }
+ }).start();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to start hive metastore server.", e);
+ }
+ }
+
+ @Override
+ public void contextDestroyed(ServletContextEvent sce) {
+ try {
+ if (broker != null) {
+ broker.stop();
+ }
+ } catch(Exception e) {
+ LOG.warn("Failed to stop activemq", e);
+ }
+ }
+
+ private String getShareLibPath() {
+ return File.separator + "user" + File.separator + USER + File.separator + "share/lib";
+ }
+
+ private String getTimestampDirectory() {
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
+ Date date = new Date();
+ return dateFormat.format(date).toString();
+ }
+
+ private String[] getLibActionDirectories() {
+ StringBuilder libActionDirectories = new StringBuilder();
+ File f = new File(shareLibPath);
+
+ for(File libDir : f.listFiles()) {
+ if (libDir.isDirectory()) {
+ libActionDirectories.append(libDir.getName()).append("\t");
+ }
+ }
+ String actionDirectories = libActionDirectories.toString();
+ return (actionDirectories).substring(0, actionDirectories.lastIndexOf('\t'))
+ .split("\t");
+ }
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/core-site.xml b/test-tools/hadoop-webapp/src/main/resources/core-site.xml
new file mode 100644
index 0000000..ef1558e
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/core-site.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>fs.fsext.impl</name>
+ <value>org.apache.falcon.hadoop.FileSystemExtension</value>
+ </property>
+
+ <property>
+ <name>fs.jail.impl</name>
+ <value>org.apache.falcon.hadoop.JailedFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>jail://global:00</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.${user.name}.groups</name>
+ <value>*</value>
+ <description>Allow the superuser oozie to impersonate any members of the group group1 and group2</description>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.${user.name}.hosts</name>
+ <value>*</value>
+ <description>The superuser can connect only from host1 and host2 to impersonate a user</description>
+ </property>
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>${project.build.directory}/tmp-hadoop-${user.name}</value>
+ <description>A base for other temporary directories.</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/hdfs-site.xml b/test-tools/hadoop-webapp/src/main/resources/hdfs-site.xml
new file mode 100644
index 0000000..0b929e6
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/hdfs-site.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:41110</value>
+ <description>
+ The datanode server address and port for data transfer.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:41175</value>
+ <description>
+ The datanode http server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:41120</value>
+ <description>
+ The datanode ipc server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.http.address</name>
+ <value>0.0.0.0:41170</value>
+ <description>
+ The address and the base port where the dfs namenode web ui will listen on.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/hive-site.xml b/test-tools/hadoop-webapp/src/main/resources/hive-site.xml
new file mode 100644
index 0000000..1bc8596
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/hive-site.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://localhost:49083</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.local</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>jail://global:00</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.client.socket.timeout</name>
+ <value>600</value>
+ <description>MetaStore Client socket timeout in seconds</description>
+ </property>
+
+ <property>
+ <name>hive.hmshandler.retry.attempts</name>
+ <value>3</value>
+ <description>The number of times to retry a HMSHandler call if there were a connection error</description>
+ </property>
+
+ <property>
+ <name>hive.hmshandler.retry.interval</name>
+ <value>1000</value>
+ <description>The number of milliseconds between HMSHandler retry attempts</description>
+ </property>
+
+ <!-- Forcing the creation of the db dir under target -->
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=./target/metastore_db;create=true</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/log4j.xml b/test-tools/hadoop-webapp/src/main/resources/log4j.xml
new file mode 100644
index 0000000..d69e921
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/log4j.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${user.dir}/target/logs/jetty.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="debug"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <root>
+ <priority value="info"/>
+ <appender-ref ref="FILE"/>
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/mapred-site.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/mapred-site.xml b/test-tools/hadoop-webapp/src/main/resources/mapred-site.xml
new file mode 100644
index 0000000..cf297de
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/mapred-site.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>mapreduce.jobtracker.address</name>
+ <value>localhost:41021</value>
+ </property>
+
+ <property>
+ <name>mapred.tasktracker.map.tasks.maximum</name>
+ <value>2</value>
+ <description>The maximum number of map tasks that will be run
+ simultaneously by a task tracker.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.tasktracker.reduce.tasks.maximum</name>
+ <value>1</value>
+ <description>The maximum number of reduce tasks that will be run
+ simultaneously by a task tracker.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.staging.root.dir</name>
+ <value>/user</value>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker.http.address</name>
+ <value>0.0.0.0:41130</value>
+ <description>
+ The job tracker http server address and port the server will listen on.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.task.tracker.http.address</name>
+ <value>0.0.0.0:41160</value>
+ <description>
+ The task tracker http server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/resources/yarn-site.xml b/test-tools/hadoop-webapp/src/main/resources/yarn-site.xml
new file mode 100644
index 0000000..658752b
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/resources/yarn-site.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <description>The address of the applications manager interface in the RM.</description>
+ <name>yarn.resourcemanager.address</name>
+ <value>0.0.0.0:41021</value>
+ </property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>unittests</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>0.0.0.0:41025</value>
+ </property>
+
+ <property>
+ <description>The address of the RM web application.</description>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>0.0.0.0:41130</value>
+ </property>
+
+ <property>
+ <description>The address of the scheduler interface.</description>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>0.0.0.0:41030</value>
+ </property>
+
+ <property>
+ <description>Address where the localizer IPC is.</description>
+ <name>yarn.nodemanager.localizer.address</name>
+ <value>0.0.0.0:41040</value>
+ </property>
+
+ <property>
+ <description>The address of the RM admin interface.</description>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>0.0.0.0:41140</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/main/webapp/WEB-INF/web.xml b/test-tools/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..8b186ad
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
+ "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+
+ <display-name>Apache Falcon Embedded Hadoop - Test Cluster</display-name>
+ <description>Apache Falcon Embedded Hadoop - Test Cluster</description>
+
+ <listener>
+ <listener-class>org.apache.falcon.listener.HadoopStartupListener</listener-class>
+ </listener>
+
+</web-app>
http://git-wip-us.apache.org/repos/asf/falcon/blob/d2fd49e5/test-tools/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java b/test-tools/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
new file mode 100644
index 0000000..193fab0
--- /dev/null
+++ b/test-tools/hadoop-webapp/src/test/java/org/apache/hadoop/mapred/LocalRunnerTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.testng.annotations.Test;
+
+/**
+ * Test for LocalRunner.
+ */
+@Test (enabled = false)
+public class LocalRunnerTest {
+
+ @SuppressWarnings("unchecked")
+ public void testLocalRunner() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("mapreduce.jobtracker.address", "localhost:41021");
+ conf.set("mapreduce.framework.name", "unittests");
+ JobClient client = new JobClient(new JobConf(conf));
+ System.out.println(client.getSystemDir());
+ }
+}