You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sh...@apache.org on 2013/08/28 13:27:06 UTC
git commit: FALCON-63 Ability to ingest hadoop libs to falcon release
package. Contributed by Suhas Vasu
Updated Branches:
refs/heads/master 16972af86 -> 19ba0368e
FALCON-63 Ability to ingest hadoop libs to falcon release package. Contributed by Suhas Vasu
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/19ba0368
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/19ba0368
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/19ba0368
Branch: refs/heads/master
Commit: 19ba0368ea6f9bbb960781ca7dbafd93b084ad00
Parents: 16972af
Author: Shwetha GS <sh...@inmobi.com>
Authored: Wed Aug 28 17:01:48 2013 +0530
Committer: Shwetha GS <sh...@inmobi.com>
Committed: Wed Aug 28 17:01:48 2013 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +
hadoop-dependencies/pom.xml | 87 +++++++++++++++++++++++++++++
pom.xml | 1 +
prism/pom.xml | 37 ++++++++++++
src/bin/service-start.sh | 19 +++++++
src/main/assemblies/bin-package.xml | 5 ++
src/main/assemblies/prism-package.xml | 5 ++
src/main/assemblies/server-package.xml | 5 ++
webapp/pom.xml | 35 ++++++++++++
9 files changed, 197 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 12188bb..675da6c 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,9 @@ Trunk (Unreleased)
via Srikanth Sundarrajan)
IMPROVEMENTS
+ FALCON-63 Ability to ingest hadoop libs to falcon release package.
+ (Suhas Vasu via Shwetha GS)
+
FALCON-83 Option to build distributed packages. (Shwetha GS via
Srikanth Sundarrajan)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/hadoop-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dependencies/pom.xml b/hadoop-dependencies/pom.xml
new file mode 100644
index 0000000..98f5a29
--- /dev/null
+++ b/hadoop-dependencies/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-main</artifactId>
+ <version>0.4-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>falcon-hadoop-dependencies</artifactId>
+ <description>Apache Falcon Hadoop Dependencies Module</description>
+ <name>Apache Falcon Hadoop Dependencies</name>
+
+ <profiles>
+ <profile>
+ <id>hadoop-1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>hadoop.profile</name>
+ <value>1</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/dependency</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 53149b0..aeba853 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,6 +147,7 @@
<module>metrics</module>
<module>common</module>
<module>test-util</module>
+ <module>hadoop-dependencies</module>
<module>hadoop-webapp</module>
<module>feed</module>
<module>messaging</module>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/prism/pom.xml
----------------------------------------------------------------------
diff --git a/prism/pom.xml b/prism/pom.xml
index 55439c1..2d78ffd 100644
--- a/prism/pom.xml
+++ b/prism/pom.xml
@@ -173,4 +173,41 @@
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>hadoop-1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>hadoop.profile</name>
+ <value>1</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/bin/service-start.sh
----------------------------------------------------------------------
diff --git a/src/bin/service-start.sh b/src/bin/service-start.sh
index a117d3f..5ebcbdf 100755
--- a/src/bin/service-start.sh
+++ b/src/bin/service-start.sh
@@ -56,6 +56,23 @@ for i in "${BASEDIR}/server/webapp/$APP_TYPE/WEB-INF/lib/"*.jar; do
FALCONCPPATH="${FALCONCPPATH}:$i"
done
+HADOOPDIR=`which hadoop`
+if [ "$HADOOPDIR" != "" ]; then
+ echo "Hadoop is installed, adding hadoop classpath to falcon classpath"
+ FALCONCPPATH="${FALCONCPPATH}:`hadoop classpath`"
+elif [ "$HADOOP_HOME" != "" ]; then
+ echo "Hadoop home is set, adding ${HADOOP_HOME}/lib/* into falcon classpath"
+ for i in "${HADOOP_HOME}/lib/"*.jar; do
+ FALCONCPPATH="${FALCONCPPATH}:$i"
+ done
+else
+ echo "Could not find installed hadoop and HADOOP_HOME is not set."
+ echo "Using the default jars bundled in ${BASEDIR}/hadooplibs/"
+ for i in "${BASEDIR}/hadooplibs/"*.jar; do
+ FALCONCPPATH="${FALCONCPPATH}:$i"
+ done
+fi
+
if [ -z "$FALCON_CONF" ]; then
CONF_PATH=${BASEDIR}/conf
else
@@ -75,3 +92,5 @@ TIME=`date +%Y%m%d%H%M%s`
nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.Main -app ${BASEDIR}/server/webapp/*.war $* 2> ${BASEDIR}/logs/$APP_TYPE.out.$TIME &
echo $! > ${BASEDIR}/logs/$APP_TYPE.pid
popd > /dev/null
+
+echo "Falcon started using hadoop version: " `${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.hadoop.util.VersionInfo | head -1`
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/bin-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/bin-package.xml b/src/main/assemblies/bin-package.xml
index 28d0449..92efad8 100644
--- a/src/main/assemblies/bin-package.xml
+++ b/src/main/assemblies/bin-package.xml
@@ -78,6 +78,11 @@
<include>README</include>
</includes>
</fileSet>
+
+ <fileSet>
+ <directory>hadoop-dependencies/target/dependency</directory>
+ <outputDirectory>hadooplibs</outputDirectory>
+ </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/prism-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/prism-package.xml b/src/main/assemblies/prism-package.xml
index 943f387..434d8bb 100644
--- a/src/main/assemblies/prism-package.xml
+++ b/src/main/assemblies/prism-package.xml
@@ -79,6 +79,11 @@
<include>README</include>
</includes>
</fileSet>
+
+ <fileSet>
+ <directory>hadoop-dependencies/target/dependency</directory>
+ <outputDirectory>hadooplibs</outputDirectory>
+ </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/server-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/server-package.xml b/src/main/assemblies/server-package.xml
index c62218b..c92ad17 100644
--- a/src/main/assemblies/server-package.xml
+++ b/src/main/assemblies/server-package.xml
@@ -78,6 +78,11 @@
<include>README</include>
</includes>
</fileSet>
+
+ <fileSet>
+ <directory>hadoop-dependencies/target/dependency</directory>
+ <outputDirectory>hadooplibs</outputDirectory>
+ </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 2b3622b..ffeeb46 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -407,6 +407,41 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>hadoop-1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>hadoop.profile</name>
+ <value>1</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>