You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/08/26 11:49:09 UTC
git commit: FALCON-83 Option to build distributed packages.
Contributed by Shwetha GS
Updated Branches:
refs/heads/master 31c62be85 -> 4f86c0b14
FALCON-83 Option to build distributed packages. Contributed by Shwetha GS
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4f86c0b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4f86c0b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4f86c0b1
Branch: refs/heads/master
Commit: 4f86c0b1494c2271605a84b4444de84478c0ceed
Parents: 31c62be
Author: srikanth.sundarrajan <sr...@inmobi.com>
Authored: Mon Aug 26 15:18:03 2013 +0530
Committer: srikanth.sundarrajan <sr...@inmobi.com>
Committed: Mon Aug 26 15:18:03 2013 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +
README | 122 +++++-------
.../falcon/service/LogCleanupService.java | 17 +-
docs/src/site/twiki/InstallationSteps.twiki | 64 ++++++-
.../falcon/listener/HadoopStartupListener.java | 30 +--
pom.xml | 192 +++++++++----------
prism/src/main/java/org/apache/falcon/Main.java | 91 +++++++++
.../src/main/java/org/apache/falcon/Prism.java | 52 -----
.../org/apache/falcon/util/EmbeddedServer.java | 4 +-
src/bin/falcon | 7 +-
src/bin/falcon-start | 28 +--
src/bin/falcon-stop | 10 +-
src/bin/prism-start | 39 ++++
src/bin/prism-stop | 34 ++++
src/bin/service-start.sh | 77 ++++++++
src/bin/service-stop.sh | 40 ++++
src/conf/client.properties | 7 +-
src/conf/prism-client.properties | 24 +++
src/conf/runtime.properties | 14 +-
src/conf/startup.properties | 16 +-
src/main/assemblies/bin-package.xml | 91 +++++++++
src/main/assemblies/client-package.xml | 73 +++++++
src/main/assemblies/package.xml | 108 -----------
src/main/assemblies/prism-package.xml | 97 ++++++++++
src/main/assemblies/server-package.xml | 91 +++++++++
src/main/assemblies/src-package.xml | 1 +
webapp/pom.xml | 13 +-
.../src/main/java/org/apache/falcon/Main.java | 95 ---------
28 files changed, 922 insertions(+), 518 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e967a85..ac8b19f 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,9 @@ Trunk (Unreleased)
via Srikanth Sundarrajan)
IMPROVEMENTS
+ FALCON-83 Option to build distributed packages. (Shwetha GS via
+ Srikanth Sundarrajan)
+
FALCON-73 Remove JUnit dependency as its used only for asserts while
TestNG for tests. (Srikanth Sundarrajan)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/README
----------------------------------------------------------------------
diff --git a/README b/README
index 79b12eb..a39f914 100644
--- a/README
+++ b/README
@@ -73,105 +73,75 @@ g. Once scheduled process instances can be managed through irovy CLI
Example configurations
Cluster:
-
-<?xml version="1.0"?>
-<!--
- Production cluster configuration
- -->
-
-<cluster colo="ua2" description="" name="staging-red" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<?xml version="1.0" encoding="UTF-8"?>
+<cluster colo="local" description="" name="local" xmlns="uri:falcon:cluster:0.1">
<interfaces>
- <interface type="readonly" endpoint="hftp://gsgw1001.red.ua2.inmobi.com:50070"
- version="0.20.2-cdh3u0" />
-
- <interface type="write" endpoint="hdfs://gsgw1001.red.ua2.inmobi.com:54310"
- version="0.20.2-cdh3u0" />
-
- <interface type="execute" endpoint="gsgw1001.red.ua2.inmobi.com:54311" version="0.20.2-cdh3u0" />
-
- <interface type="workflow" endpoint="http://gs1134.blue.ua2.inmobi.com:11000/oozie/"
- version="3.1.4" />
-
- <interface type="messaging" endpoint="tcp://gs1134.blue.ua2.inmobi.com:61616?daemon=true"
- version="5.1.6" />
+ <interface type="readonly" endpoint="hftp://localhost:41110"
+ version="0.20.2"/>
+ <interface type="write" endpoint="hdfs://localhost:41020"
+ version="0.20.2"/>
+ <interface type="execute" endpoint="localhost:41021" version="0.20.2"/>
+ <interface type="workflow" endpoint="http://localhost:41000/oozie/"
+ version="3.1"/>
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"
+ version="5.1.6"/>
+ <interface type="registry" endpoint="Hcat" version="1"/>
</interfaces>
-
<locations>
- <location name="staging" path="/projects/falcon/staging" />
- <location name="temp" path="/tmp" />
- <location name="working" path="/projects/falcon/working" />
+ <location name="staging" path="/projects/falcon/staging"/>
+ <location name="temp" path="/tmp"/>
+ <location name="working" path="/project/falcon/working"/>
</locations>
-
- <properties/>
-
</cluster>
Feed:
-
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Hourly ad carrier summary. Generated by hourly processing of rr logs
- -->
-
-<feed description="RRHourlyAdCarrierSummary" name="RRHourlyAdCarrierSummary" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <partitions/>
+<feed description="in" name="in" xmlns="uri:ivory:feed:0.1">
+ <partitions>
+ <partition name="type"/>
+ </partitions>
+ <groups>in</groups>
- <groups>rmchourly</groups>
-
- <frequency>hours</frequency>
- <periodicity>1</periodicity>
-
- <late-arrival cut-off="hours(6)" />
+ <frequency>hours(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)"/>
<clusters>
- <cluster name="staging-red" type="source">
- <validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" timezone="UTC" />
- <retention limit="months(24)" action="delete" />
+ <cluster name="local">
+ <validity start="2013-01-01T00:00Z" end="2020-01-01T12:00Z"/>
+ <retention limit="hours(24)" action="delete"/>
</cluster>
</clusters>
<locations>
- <location type="data" path="/projects/bi/rmc/rr/${YEAR}-${MONTH}-${DAY}-${HOUR}.concat/HourlyAdCarrierSummary" />
- <location type="stats" path="/none" />
- <location type="meta" path="/none" />
+ <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}"/>
</locations>
- <ACL owner="rmcuser" group="users" permission="0755" />
-
- <schema location="/none" provider="none" />
-
- <properties/>
-
+ <ACL owner="testuser" group="group" permission="0x644"/>
+ <schema location="/schema/in/in.format.csv" provider="csv"/>
</feed>
-
Process:
-
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- RMC Daily process, produces 34 new feeds
- -->
-<process name="rmc-daily">
- <cluster name="staging-red" />
-
- <frequency>days(1)</frequency>
-
- <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" timezone="UTC" />
-
- <inputs>
- <input name="WapAd" feed="WapAd" start="today(0,0)" end="today(0,0)" />
- </inputs>
+<process name="wf-process" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="local">
+ <validity start="2013-01-01T00:00Z" end="2013-01-01T02:00Z"/>
+ </cluster>
+ </clusters>
- <outputs>
- <output name="TrafficDailyAdSiteSummary" feed="TrafficDailyAdSiteSummary" instance="yesterday(0,0)" />
- </outputs>
+ <parallel>1</parallel>
+ <order>LIFO</order>
+ <frequency>hours(1)</frequency>
+ <timezone>UTC</timezone>
- <properties>
- <property name="lastday" value="${formatTime(yesterday(0,0), 'yyyy-MM-dd')}" />
- </properties>
+ <inputs>
+ <input name="input" feed="in" start="now(0,0)" end="now(0,0)"/>
+ </inputs>
- <workflow engine="oozie" path="/projects/bi/rmc/pipelines/workflow/rmcdaily" />
+ <outputs>
+ <output name="output" feed="out" instance="today(0,0)"/>
+ </outputs>
- <retry policy="backoff" delay="5" delayUnit="minutes" attempts="3" />
+ <workflow engine="oozie" path="/app/mapred-wf.xml"/>
</process>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/common/src/main/java/org/apache/falcon/service/LogCleanupService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/service/LogCleanupService.java b/common/src/main/java/org/apache/falcon/service/LogCleanupService.java
index ca646d5..689c8a0 100644
--- a/common/src/main/java/org/apache/falcon/service/LogCleanupService.java
+++ b/common/src/main/java/org/apache/falcon/service/LogCleanupService.java
@@ -17,6 +17,13 @@
*/
package org.apache.falcon.service;
+import java.util.Date;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import javax.servlet.jsp.el.ELException;
+import javax.servlet.jsp.el.ExpressionEvaluator;
+
import org.apache.commons.el.ExpressionEvaluatorImpl;
import org.apache.falcon.FalconException;
import org.apache.falcon.aspect.GenericAlert;
@@ -27,12 +34,6 @@ import org.apache.falcon.expression.ExpressionHelper;
import org.apache.falcon.util.StartupProperties;
import org.apache.log4j.Logger;
-import javax.servlet.jsp.el.ELException;
-import javax.servlet.jsp.el.ExpressionEvaluator;
-import java.util.Date;
-import java.util.Timer;
-import java.util.TimerTask;
-
/**
* Log cleanup service.
*/
@@ -57,8 +58,8 @@ public class LogCleanupService implements FalconService {
private static class CleanupThread extends TimerTask {
- private AbstractCleanupHandler processCleanupHandler = new ProcessCleanupHandler();
- private AbstractCleanupHandler feedCleanupHandler = new FeedCleanupHandler();
+ private final AbstractCleanupHandler processCleanupHandler = new ProcessCleanupHandler();
+ private final AbstractCleanupHandler feedCleanupHandler = new FeedCleanupHandler();
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 139ee8e..cf55697 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -15,15 +15,16 @@ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
</verbatim>
-Once the build successfully completes, artifacts can be packaged for deployment. By default the falcon is built for embedded mode.
+Once the build successfully completes, artifacts can be packaged for deployment. The package can be built in embedded or distributed mode.
+*Embedded Mode*
<verbatim>
-mvn clean assembly:assembly -DskipTests
+mvn clean assembly:assembly -DskipTests -DskipCheck=true
</verbatim>
-Tar can be found in {project dir}/target/falcon-${project.version}.tar.gz
+Tar can be found in {project dir}/target/falcon-${project.version}-bin.tar.gz
Tar is structured as follows
@@ -40,31 +41,71 @@ Tar is structured as follows
|- log4j.xml
|- src
|- docs
+|- apidocs
|- client
|- lib (client support libs)
|- server
|- webapp
- |- classes (serer support classes)
- |- lib (server support libs)
- |- falcon.war
+ |- falcon.war
|- logs (application log files & temp data files)
|- falcon.pid
+
+</verbatim>
+
+*Distributed Mode*
+
+<verbatim>
+
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed
</verbatim>
+This generates 3 tars - {project dir}/target/falcon-${project.version}-prism.tar.gz, {project dir}/target/falcon-${project.version}-server.tar.gz and {project dir}/target/falcon-${project.version}-client.tar.gz
+
+Package structure of prism and server tars is same as that of the embedded package with one difference that prism package contains prism war instead of falcon war. Client package is structured as follows
+
+<verbatim>
+
+|- bin
+ |- falcon
+|- conf
+ |- client.properties
+ |- log4j.xml
+|- client
+ |- lib (client support libs)
+
+</verbatim>
+
---+++ Installing & running Falcon
*Installing falcon*
<verbatim>
-tar -xzvf falcon-${project.version}.tar.gz
-cd falcon-main-${project.version}
+tar -xzvf {falcon package}
+cd falcon-server-${project.version} or cd falcon-prism-${project.version}
</verbatim>
*Starting Falcon Server*
<verbatim>
-bin/falcon-start
+bin/falcon-start [-port <port>]
+</verbatim>
+
+By default,
+- falcon server starts at port 15000. To change the port, use -port option
+- falcon server starts embedded active mq. To control this behaviour, set the following system properties using -D option in environment variable FALCON_OPTS:
+ - falcon.embeddedmq=<true/false> - Should server start embedded active mq, default true
+ - falcon.emeddedmq.port=<port> - Port for embedded active mq, default 61616
+ - falcon.embeddedmq.data=<path> - Data path for embedded active mq, default {package dir}/logs/data
+- falcon server starts with conf from {package dir}/conf. To override this (to use the same conf with multiple falcon upgrades), set environment variable FALCON_CONF to the path of conf dir
+
+*Starting Prism*
+<verbatim>
+bin/prism-start [-port <port>]
</verbatim>
+By default,
+- falcon server starts at port 16000. To change the port, use -port option
+- prism starts with conf from {package dir}/conf. To override this (to use the same conf with multiple prism upgrades), set environment variable FALCON_CONF to the path of conf dir
+
*Using Falcon*
<verbatim>
bin/falcon admin -version
@@ -81,6 +122,11 @@ bin/falcon help
bin/falcon-stop
</verbatim>
+*Stopping Prism*
+<verbatim>
+bin/prism-stop
+</verbatim>
+
---+++ Preparing oozie bundle for use with Falcon
<verbatim>
cd <<project home>>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java b/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
index 1ce709d..5114a4b 100644
--- a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
+++ b/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
@@ -51,10 +51,13 @@ public class HadoopStartupListener implements ServletContextListener {
NameNode.createNameNode(emptyArgs, conf);
DataNode.createDataNode(emptyArgs, conf);
JobConf jobConf = new JobConf(conf);
-// JobTracker jt = JobTracker.startTracker(jobConf);
-// jt.offerService();
-// TaskTracker tt = new TaskTracker(jobConf);
-// tt.run();
+ /**
+ * Reflection code:
+ * JobTracker jt = JobTracker.startTracker(jobConf);
+ * jt.offerService();
+ * TaskTracker tt = new TaskTracker(jobConf);
+ * tt.run();
+ */
Object jt = Class.forName("org.apache.hadoop.mapred.JobTracker")
.getMethod("startTracker", JobConf.class).invoke(null, jobConf);
startService(jt, "offerService");
@@ -62,14 +65,17 @@ public class HadoopStartupListener implements ServletContextListener {
.getConstructor(JobConf.class).newInstance(jobConf);
startService(tt, "run");
} else if (hadoopProfle.equals("2")) {
-// DefaultMetricsSystem.setMiniClusterMode(true);
-// ResourceManager resourceManager = new ResourceManager(new MemStore());
-// YarnConfiguration yarnConf = new YarnConfiguration(conf);
-// resourceManager.init(yarnConf);
-// resourceManager.start();
-// NodeManager nodeManager = new NodeManager();
-// nodeManager.init(yarnConf);
-// nodeManager.start();
+ /**
+ * Reflection code:
+ * DefaultMetricsSystem.setMiniClusterMode(true);
+ * ResourceManager resourceManager = new ResourceManager(new MemStore());
+ * YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ * resourceManager.init(yarnConf);
+ * resourceManager.start();
+ * NodeManager nodeManager = new NodeManager();
+ * nodeManager.init(yarnConf);
+ * nodeManager.start();
+ */
Class.forName("org.apache.hadoop.metrics2.lib.DefaultMetricsSystem")
.getMethod("setMiniClusterMode", boolean.class).invoke(null, true);
NameNode.createNameNode(emptyArgs, conf);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7668bb9..53149b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,6 +96,8 @@
<include.prism>true</include.prism>
+ <hadoop.profile>1</hadoop.profile>
+ <hadoop.version>1.1.2</hadoop.version>
<slf4j.version>1.2</slf4j.version>
<oozie.version>3.2.2</oozie.version>
<activemq.version>5.4.3</activemq.version>
@@ -107,60 +109,6 @@
<profiles>
<profile>
- <id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- <property>
- <name>hadoop.profile</name>
- <value>1</value>
- </property>
- </activation>
- <properties>
- <hadoop.profile>1</hadoop.profile>
- <hadoop.version>1.1.2</hadoop.version>
- </properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jdt</groupId>
- <artifactId>core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.ftpserver</groupId>
- <artifactId>ftpserver-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ftpserver</groupId>
- <artifactId>ftpserver-deprecated</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </dependencyManagement>
- </profile>
-
- <profile>
<id>hadoop-2</id>
<activation>
<property>
@@ -172,47 +120,25 @@
<hadoop.profile>2</hadoop.profile>
<hadoop.version>2.0.2-alpha</hadoop.version>
</properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <classifier>tests</classifier>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <classifier>tests</classifier>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
+ </profile>
+
+ <profile>
+ <id>distributed</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/prism-package.xml</descriptor>
+ <descriptor>src/main/assemblies/server-package.xml</descriptor>
+ <descriptor>src/main/assemblies/client-package.xml</descriptor>
+ </descriptors>
+ <finalName>falcon-${project.version}</finalName>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</profile>
</profiles>
@@ -295,6 +221,80 @@
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jdt</groupId>
+ <artifactId>core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.ftpserver</groupId>
+ <artifactId>ftpserver-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ftpserver</groupId>
+ <artifactId>ftpserver-deprecated</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1</version>
@@ -899,7 +899,7 @@
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
- <descriptor>src/main/assemblies/package.xml</descriptor>
+ <descriptor>src/main/assemblies/bin-package.xml</descriptor>
<descriptor>src/main/assemblies/src-package.xml</descriptor>
</descriptors>
<finalName>falcon-${project.version}</finalName>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/prism/src/main/java/org/apache/falcon/Main.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/Main.java b/prism/src/main/java/org/apache/falcon/Main.java
new file mode 100644
index 0000000..b59983f
--- /dev/null
+++ b/prism/src/main/java/org/apache/falcon/Main.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon;
+
+import org.apache.activemq.broker.BrokerService;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.falcon.util.BuildProperties;
+import org.apache.falcon.util.EmbeddedServer;
+import org.apache.log4j.Logger;
+
+/**
+ * Driver for running Falcon as a standalone server with embedded jetty server.
+ */
+public final class Main {
+ private static final Logger LOG = Logger.getLogger(Main.class);
+ private static final String APP_PATH = "app";
+ private static final String APP_PORT = "port";
+
+ /**
+ * Prevent users from constructing this.
+ */
+ private Main() {
+ }
+
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+ Option opt;
+
+ opt = new Option(APP_PATH, true, "Application Path");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option(APP_PORT, true, "Application Port");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ return new GnuParser().parse(options, args);
+ }
+
+ public static void main(String[] args) throws Exception {
+ CommandLine cmd = parseArgs(args);
+ String projectVersion = BuildProperties.get().getProperty("project.version");
+ String appPath = "webapp/target/falcon-webapp-" + projectVersion;
+ int appPort = 15000;
+
+ if (cmd.hasOption(APP_PATH)) {
+ appPath = cmd.getOptionValue(APP_PATH);
+ }
+
+ if (cmd.hasOption(APP_PORT)) {
+ appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
+ }
+
+ boolean startActiveMq = Boolean.valueOf(System.getProperty("falcon.embeddedmq", "true"));
+ if (startActiveMq) {
+ String dataDir = System.getProperty("falcon.embeddedmq.data", "target/");
+ int mqport = Integer.valueOf(System.getProperty("falcon.emeddedmq.port", "61616"));
+ LOG.info("Starting activemq at port " + mqport + " with data dir " + dataDir);
+
+ BrokerService broker = new BrokerService();
+ broker.setUseJmx(false);
+ broker.setDataDirectory(dataDir);
+ broker.addConnector("vm://localhost");
+ broker.addConnector("tcp://localhost:" + mqport);
+ broker.start();
+ }
+
+ EmbeddedServer server = new EmbeddedServer(appPort, appPath);
+ server.start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/prism/src/main/java/org/apache/falcon/Prism.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/Prism.java b/prism/src/main/java/org/apache/falcon/Prism.java
deleted file mode 100644
index d67ecb9..0000000
--- a/prism/src/main/java/org/apache/falcon/Prism.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon;
-
-import org.apache.falcon.util.BuildProperties;
-import org.apache.falcon.util.EmbeddedServer;
-
-/**
- * Driver for running Prism as a standalone server.
- */
-public final class Prism {
-
- /**
- * Default port number for embedded server.
- */
- private static final int PORT = 16000;
-
- /**
- * Prevent users from constructing this.
- */
- private Prism() {
- }
-
- /**
- * Main method.
- *
- * @param args argument array
- * @throws Exception exception
- */
- public static void main(final String[] args) throws Exception {
- String projectVersion = BuildProperties.get().getProperty("project.version");
- EmbeddedServer server = new EmbeddedServer(PORT,
- "prism/target/falcon-prism-" + projectVersion);
- server.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/prism/src/main/java/org/apache/falcon/util/EmbeddedServer.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/util/EmbeddedServer.java b/prism/src/main/java/org/apache/falcon/util/EmbeddedServer.java
index 80c6d80..e1b5ec8 100644
--- a/prism/src/main/java/org/apache/falcon/util/EmbeddedServer.java
+++ b/prism/src/main/java/org/apache/falcon/util/EmbeddedServer.java
@@ -28,8 +28,8 @@ import org.mortbay.jetty.webapp.WebAppContext;
* This class embeds a Jetty server and a connector.
*/
public class EmbeddedServer {
- private Server server = new Server();
- private Connector connector = new SocketConnector();
+ private final Server server = new Server();
+ private final Connector connector = new SocketConnector();
public EmbeddedServer(int port, String path) {
connector.setPort(port);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/falcon
----------------------------------------------------------------------
diff --git a/src/bin/falcon b/src/bin/falcon
index 5e290f6..696f606 100755
--- a/src/bin/falcon
+++ b/src/bin/falcon
@@ -29,10 +29,13 @@ done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
-FALCONCPPATH="${BASEDIR}/conf:"
+FALCONCPPATH="$FALCON_CONF:${BASEDIR}/conf:"
for i in "${BASEDIR}/client/lib/"*.jar; do
FALCONCPPATH="${FALCONCPPATH}:$i"
done
+for i in "${BASEDIR}/server/webapp/"*/WEB-INF/lib/*.jar; do
+ FALCONCPPATH="${FALCONCPPATH}:$i"
+done
if test -z ${JAVA_HOME}
@@ -42,7 +45,7 @@ else
JAVA_BIN=${JAVA_HOME}/bin/java
fi
-JAVA_PROPERTIES=""
+JAVA_PROPERTIES="$FALCON_OPTS"
while [[ ${1} =~ ^\-D ]]; do
JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
shift
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/falcon-start
----------------------------------------------------------------------
diff --git a/src/bin/falcon-start b/src/bin/falcon-start
index f7b6524..b29cb4d 100755
--- a/src/bin/falcon-start
+++ b/src/bin/falcon-start
@@ -13,6 +13,8 @@
# limitations under the License. See accompanying LICENSE file.
#
+set -e
+
# resolve links - $0 may be a softlink
PRG="${0}"
@@ -29,28 +31,4 @@ done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
-FALCONCPPATH="${BASEDIR}/conf:${BASEDIR}/server/webapp/classes:"
-for i in "${BASEDIR}/server/webapp/lib/"*.jar; do
- FALCONCPPATH="${FALCONCPPATH}:$i"
-done
-
-mkdir -p ${BASEDIR}/logs
-
-if test -z ${JAVA_HOME}
-then
- JAVA_BIN=java
-else
- JAVA_BIN=${JAVA_HOME}/bin/java
-fi
-
-pushd ${BASEDIR} > /dev/null
-JAVA_PROPERTIES="$FALCON_OPTS -Dconfig.location=${BASEDIR}/conf"
-while [[ ${1} =~ ^\-D ]]; do
- JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
- shift
-done
-TIME=`date +%Y%m%d%H%M%s`
-nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.Main -app ${BASEDIR}/server/webapp/falcon.war -mqbase ${BASEDIR}/logs/data $* 2> ${BASEDIR}/logs/falcon.out.$TIME &
-echo $! > ${BASEDIR}/logs/falcon.pid
-popd > /dev/null
-
+exec ${BASEDIR}/bin/service-start.sh falcon $ARGS $*
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/falcon-stop
----------------------------------------------------------------------
diff --git a/src/bin/falcon-stop b/src/bin/falcon-stop
index dbfc352..45f2056 100755
--- a/src/bin/falcon-stop
+++ b/src/bin/falcon-stop
@@ -13,6 +13,8 @@
# limitations under the License. See accompanying LICENSE file.
#
+set -e
+
# resolve links - $0 may be a softlink
PRG="${0}"
@@ -29,10 +31,4 @@ done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
-if [ -f ${BASEDIR}/logs/falcon.pid ]
-then
- kill -15 `cat ${BASEDIR}/logs/falcon.pid`
-else
- echo "pid file ${BASEDIR}/logs/falcon.pid not present"
-fi
-
+exec ${BASEDIR}/bin/service-stop.sh falcon
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/prism-start
----------------------------------------------------------------------
diff --git a/src/bin/prism-start b/src/bin/prism-start
new file mode 100755
index 0000000..3c8392a
--- /dev/null
+++ b/src/bin/prism-start
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+set -e
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+if [[ $* != *-port* ]]; then
+ ARGS="-port 16000"
+fi
+
+export FALCON_PROPERTIES="-Dfalcon.embeddedmq=false -Dfalcon.domain=prism"
+exec ${BASEDIR}/bin/service-start.sh prism $ARGS $*
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/prism-stop
----------------------------------------------------------------------
diff --git a/src/bin/prism-stop b/src/bin/prism-stop
new file mode 100755
index 0000000..e39f6e3
--- /dev/null
+++ b/src/bin/prism-stop
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+set -e
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+exec ${BASEDIR}/bin/service-stop.sh prism
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/service-start.sh
----------------------------------------------------------------------
diff --git a/src/bin/service-start.sh b/src/bin/service-start.sh
new file mode 100755
index 0000000..a117d3f
--- /dev/null
+++ b/src/bin/service-start.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+set -e
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+
+mkdir -p ${BASEDIR}/logs
+
+if test -z ${JAVA_HOME}
+then
+ JAVA_BIN=java
+else
+ JAVA_BIN=${JAVA_HOME}/bin/java
+fi
+
+pushd ${BASEDIR} > /dev/null
+
+APP_TYPE=$1
+if [ ! -d ${BASEDIR}/server/webapp/$APP_TYPE/WEB-INF ]; then
+ mkdir -p ${BASEDIR}/server/webapp/$APP_TYPE
+ cd ${BASEDIR}/server/webapp/$APP_TYPE
+ jar -xf ../$APP_TYPE.war
+ cd -
+fi
+
+FALCONCPPATH="$FALCON_CONF:${BASEDIR}/conf:${BASEDIR}/server/webapp/$APP_TYPE/WEB-INF/classes:"
+for i in "${BASEDIR}/server/webapp/$APP_TYPE/WEB-INF/lib/"*.jar; do
+ FALCONCPPATH="${FALCONCPPATH}:$i"
+done
+
+if [ -z "$FALCON_CONF" ]; then
+ CONF_PATH=${BASEDIR}/conf
+else
+ CONF_PATH=$FALCON_CONF
+fi
+
+JAVA_PROPERTIES="$FALCON_OPTS $FALCON_PROPERTIES -Dfalcon.embeddedmq.data=${BASEDIR}/logs/data -Dfalcon.home=${BASEDIR} -Dconfig.location=$CONF_PATH"
+shift
+
+while [[ ${1} =~ ^\-D ]]; do
+ JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
+ shift
+done
+TIME=`date +%Y%m%d%H%M%s`
+
+
+nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.Main -app ${BASEDIR}/server/webapp/*.war $* 2> ${BASEDIR}/logs/$APP_TYPE.out.$TIME &
+echo $! > ${BASEDIR}/logs/$APP_TYPE.pid
+popd > /dev/null
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/bin/service-stop.sh
----------------------------------------------------------------------
diff --git a/src/bin/service-stop.sh b/src/bin/service-stop.sh
new file mode 100755
index 0000000..4dc62a6
--- /dev/null
+++ b/src/bin/service-stop.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+APP_TYPE=$1
+PID_FILE=${BASEDIR}/logs/$APP_TYPE.pid
+if [ -f $PID_FILE ]
+then
+ kill -15 `cat $PID_FILE`
+else
+ echo "pid file $PID_FILE not present"
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/conf/client.properties
----------------------------------------------------------------------
diff --git a/src/conf/client.properties b/src/conf/client.properties
index 9ff0631..c3892fd 100644
--- a/src/conf/client.properties
+++ b/src/conf/client.properties
@@ -16,8 +16,9 @@
# limitations under the License.
#
-####################################################
-#### This is used for falcon packaging only. ####
-####################################################
+#########################################################################
+########## This is used for falcon packaging only. ###################
+## Uses default port. Please change if configured for non-default port ##
+#########################################################################
falcon.url=http://localhost:15000/
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/conf/prism-client.properties
----------------------------------------------------------------------
diff --git a/src/conf/prism-client.properties b/src/conf/prism-client.properties
new file mode 100644
index 0000000..3a948aa
--- /dev/null
+++ b/src/conf/prism-client.properties
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#########################################################################
+########## This is used for falcon packaging only. ###################
+## Uses default port. Please change if configured for non-default port ##
+#########################################################################
+
+falcon.url=http://localhost:16000/
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/conf/runtime.properties
----------------------------------------------------------------------
diff --git a/src/conf/runtime.properties b/src/conf/runtime.properties
index 3a1b62e..2192126 100644
--- a/src/conf/runtime.properties
+++ b/src/conf/runtime.properties
@@ -20,10 +20,14 @@
#### This is used for falcon packaging only. ####
####################################################
-*.domain=debug
+*.domain=prism
-*.log.cleanup.frequency.minutes.retention =hours(6)
-*.log.cleanup.frequency.hours.retention =minutes(1)
-*.log.cleanup.frequency.days.retention =days(7)
-*.log.cleanup.frequency.months.retention =months(3)
+*.log.cleanup.frequency.minutes.retention=hours(6)
+*.log.cleanup.frequency.hours.retention=minutes(1)
+*.log.cleanup.frequency.days.retention=days(7)
+*.log.cleanup.frequency.months.retention=months(3)
+
+#### To configure falcon servers with prism ####
+#*.all.colos=<comma separated list of colos where falcon servers are installed>
+#*.falcon.<colo>.endpoint=<falcon server endpoint>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index 7f1df68..6b8617e 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -20,7 +20,7 @@
#### This is used for falcon packaging only. ####
####################################################
-*.domain=debug
+*.domain=server
######### Implementation classes #########
## DONT MODIFY UNLESS SURE ABOUT CHANGE ##
@@ -37,19 +37,24 @@
org.apache.falcon.rerun.service.LateRunService,\
org.apache.falcon.service.SLAMonitoringService,\
org.apache.falcon.service.LogCleanupService
+prism.application.services=org.apache.falcon.entity.store.ConfigurationStore
*.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
org.apache.falcon.entity.ColoClusterRelation,\
org.apache.falcon.group.FeedGroupMap,\
org.apache.falcon.service.SharedLibraryHostingService
+prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap
*.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory
*.shared.libs=activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms,s4fs-0.1.jar
######### Implementation classes #########
-debug.config.store.uri=file://${user.dir}/store
-debug.system.lib.location=${user.dir}/server/webapp/lib
-debug.broker.url=tcp://localhost:61616
-debug.retry.recorder.path=${user.dir}/logs/retry
+*.config.store.uri=file://${falcon.home}/store
+*.system.lib.location=${falcon.home}/server/webapp/falcon/WEB-INF/lib
+prism.system.lib.location=${falcon.home}/server/webapp/prism/WEB-INF/lib
+*.broker.url=tcp://localhost:61616
+*.retry.recorder.path=${falcon.home}/logs/retry
*.falcon.cleanup.service.frequency=days(1)
@@ -57,7 +62,6 @@ debug.retry.recorder.path=${user.dir}/logs/retry
*.broker.ttlInMins=4320
*.entity.topic=FALCON.ENTITY.TOPIC
*.max.retry.failure.count=1
-*.retry.recorder.path=${user.dir}/logs/retry
######### Properties for configuring iMon client and metric #########
*.internal.queue.size=1000
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/bin-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/bin-package.xml b/src/main/assemblies/bin-package.xml
new file mode 100644
index 0000000..28d0449
--- /dev/null
+++ b/src/main/assemblies/bin-package.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <id>bin</id>
+ <baseDirectory>falcon-server-${project.version}</baseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/conf/</directory>
+ <outputDirectory>conf</outputDirectory>
+ <excludes>
+ <exclude>prism-client.properties</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>docs/target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>webapp/target/apidocs</directory>
+ <outputDirectory>apidocs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>falcon</include>
+ <include>falcon-start</include>
+ <include>falcon-stop</include>
+ <include>service-start.sh</include>
+ <include>service-stop.sh</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>logs</directory>
+ <outputDirectory>logs</outputDirectory>
+ <directoryMode>0777</directoryMode>
+ <excludes>
+ <exclude>*</exclude>
+ <exclude>**/**</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>DISCLAIMER.txt</include>
+ <include>NOTICE.txt</include>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ <include>README</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>webapp/target/falcon-webapp-${project.version}.war</source>
+ <outputDirectory>server/webapp</outputDirectory>
+ <destName>falcon.war</destName>
+ </file>
+ </files>
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/client-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/client-package.xml b/src/main/assemblies/client-package.xml
new file mode 100644
index 0000000..e0c75f5
--- /dev/null
+++ b/src/main/assemblies/client-package.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <id>client</id>
+ <baseDirectory>falcon-client-${project.version}</baseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>client/target/dependency</directory>
+ <outputDirectory>client/lib</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>docs/target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>webapp/target/apidocs</directory>
+ <outputDirectory>apidocs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>DISCLAIMER.txt</include>
+ <include>NOTICE.txt</include>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ <include>README</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>client/target/falcon-client-${project.version}.jar</source>
+ <outputDirectory>client/lib</outputDirectory>
+ </file>
+
+ <file>
+ <source>src/bin/falcon</source>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </file>
+
+ <file>
+ <source>src/conf/client.properties</source>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ </files>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/package.xml b/src/main/assemblies/package.xml
deleted file mode 100644
index be58be1..0000000
--- a/src/main/assemblies/package.xml
+++ /dev/null
@@ -1,108 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <formats>
- <format>tar.gz</format>
- </formats>
- <id>bin</id>
- <fileSets>
- <fileSet>
- <directory>src/conf/</directory>
- <outputDirectory>conf</outputDirectory>
- </fileSet>
-
- <fileSet>
- <directory>client/target/dependency</directory>
- <outputDirectory>client/lib</outputDirectory>
- </fileSet>
-
- <fileSet>
- <directory>client/target/dependency</directory>
- <outputDirectory>client/lib</outputDirectory>
- </fileSet>
-
- <fileSet>
- <directory>docs/target/site</directory>
- <outputDirectory>docs</outputDirectory>
- </fileSet>
-
- <fileSet>
- <directory>webapp/target/apidocs</directory>
- <outputDirectory>apidocs</outputDirectory>
- </fileSet>
-
- <fileSet>
- <directory>src/bin</directory>
- <outputDirectory>bin</outputDirectory>
- <fileMode>0755</fileMode>
- <directoryMode>0755</directoryMode>
- </fileSet>
-
- <fileSet>
- <directory>logs</directory>
- <outputDirectory>logs</outputDirectory>
- <directoryMode>0777</directoryMode>
- <excludes>
- <exclude>*</exclude>
- <exclude>**/**</exclude>
- </excludes>
- </fileSet>
-
- <fileSet>
- <directory>./</directory>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>DISCLAIMER.txt</include>
- <include>NOTICE.txt</include>
- <include>LICENSE.txt</include>
- <include>CHANGES.txt</include>
- <include>README</include>
- </includes>
- </fileSet>
-
- <fileSet>
- <directory>webapp/target/falcon-webapp-${project.version}/WEB-INF/classes</directory>
- <outputDirectory>server/webapp/classes</outputDirectory>
- <includes>
- <include>**/*.class</include>
- </includes>
- </fileSet>
-
- <fileSet>
- <directory>webapp/target/falcon-webapp-${project.version}/WEB-INF/lib</directory>
- <outputDirectory>server/webapp/lib</outputDirectory>
- </fileSet>
- </fileSets>
-
- <files>
- <file>
- <source>client/target/falcon-client-${project.version}.jar</source>
- <outputDirectory>client/lib</outputDirectory>
- </file>
-
- <file>
- <source>webapp/target/falcon-webapp-${project.version}.war</source>
- <outputDirectory>server/webapp</outputDirectory>
- <destName>falcon.war</destName>
- </file>
- </files>
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/prism-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/prism-package.xml b/src/main/assemblies/prism-package.xml
new file mode 100644
index 0000000..943f387
--- /dev/null
+++ b/src/main/assemblies/prism-package.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <id>prism</id>
+ <baseDirectory>falcon-prism-${project.version}</baseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/conf/</directory>
+ <outputDirectory>conf</outputDirectory>
+ <excludes>
+ <exclude>client.properties</exclude>
+ <exclude>prism-client.properties</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>docs/target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>webapp/target/apidocs</directory>
+ <outputDirectory>apidocs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>falcon</include>
+ <include>prism-start</include>
+ <include>prism-stop</include>
+ <include>service-start.sh</include>
+ <include>service-stop.sh</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>logs</directory>
+ <outputDirectory>logs</outputDirectory>
+ <directoryMode>0777</directoryMode>
+ <excludes>
+ <exclude>*</exclude>
+ <exclude>**/**</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>DISCLAIMER.txt</include>
+ <include>NOTICE.txt</include>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ <include>README</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>src/conf/prism-client.properties</source>
+ <outputDirectory>conf</outputDirectory>
+ <destName>client.properties</destName>
+ </file>
+
+ <file>
+ <source>prism/target/falcon-prism-${project.version}.war</source>
+ <outputDirectory>server/webapp</outputDirectory>
+ <destName>prism.war</destName>
+ </file>
+ </files>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/server-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/server-package.xml b/src/main/assemblies/server-package.xml
new file mode 100644
index 0000000..c62218b
--- /dev/null
+++ b/src/main/assemblies/server-package.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <id>server</id>
+ <baseDirectory>falcon-server-${project.version}</baseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/conf/</directory>
+ <outputDirectory>conf</outputDirectory>
+ <excludes>
+ <exclude>prism-client.properties</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>docs/target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>webapp/target/apidocs</directory>
+ <outputDirectory>apidocs</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>falcon</include>
+ <include>falcon-start</include>
+ <include>falcon-stop</include>
+ <include>service-start.sh</include>
+ <include>service-stop.sh</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>logs</directory>
+ <outputDirectory>logs</outputDirectory>
+ <directoryMode>0777</directoryMode>
+ <excludes>
+ <exclude>*</exclude>
+ <exclude>**/**</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>DISCLAIMER.txt</include>
+ <include>NOTICE.txt</include>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ <include>README</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>webapp/target/falcon-webapp-${project.version}.war</source>
+ <outputDirectory>server/webapp</outputDirectory>
+ <destName>falcon.war</destName>
+ </file>
+ </files>
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/src/main/assemblies/src-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/src-package.xml b/src/main/assemblies/src-package.xml
index 8fa7b7c..677d66e 100644
--- a/src/main/assemblies/src-package.xml
+++ b/src/main/assemblies/src-package.xml
@@ -23,6 +23,7 @@
<format>tar.gz</format>
</formats>
<id>sources</id>
+ <baseDirectory>falcon-sources-${project.version}</baseDirectory>
<fileSets>
<fileSet>
<directory>.</directory>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 52e36f7..2e698f0 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -297,6 +297,7 @@
<configuration>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<forkMode>always</forkMode>
+ <argLine>-Djava.security.krb5.realm= -Djava.security.krb5.kdc=</argLine>
</configuration>
<executions>
<execution>
@@ -412,18 +413,6 @@
</plugins>
</build>
</profile>
-
- <profile>
- <id>hadoop-2</id>
- <activation>
- <property>
- <name>hadoop.profile</name>
- <value>2</value>
- </property>
- </activation>
- <dependencies>
- </dependencies>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f86c0b1/webapp/src/main/java/org/apache/falcon/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/Main.java b/webapp/src/main/java/org/apache/falcon/Main.java
deleted file mode 100644
index 3ffe220..0000000
--- a/webapp/src/main/java/org/apache/falcon/Main.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon;
-
-import org.apache.activemq.broker.BrokerService;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.falcon.util.BuildProperties;
-import org.apache.falcon.util.EmbeddedServer;
-
-/**
- * Driver for running Falcon as a standalone server with embedded jetty server.
- */
-public final class Main {
-
- private static final String APP_PATH = "app";
- private static final String EMBEDDED_ACTIVEMQ = "embeddedmq";
- private static final String ACTIVEMQ_BASE = "mqbase";
-
- /**
- * Prevent users from constructing this.
- */
- private Main() {
- }
-
- private static CommandLine parseArgs(String[] args) throws ParseException {
- Options options = new Options();
- Option opt;
-
- opt = new Option(APP_PATH, true, "Application Path");
- opt.setRequired(false);
- options.addOption(opt);
-
- opt = new Option(EMBEDDED_ACTIVEMQ, true, "Should start embedded activemq?");
- opt.setRequired(false);
- options.addOption(opt);
-
- opt = new Option(ACTIVEMQ_BASE, true, "Activemq data directory");
- opt.setRequired(false);
- options.addOption(opt);
-
- return new GnuParser().parse(options, args);
- }
-
- public static void main(String[] args) throws Exception {
- CommandLine cmd = parseArgs(args);
- String projectVersion = BuildProperties.get().getProperty("project.version");
- String appPath = "webapp/target/falcon-webapp-" + projectVersion;
- String dataDir = "target/";
- boolean startActiveMq = true;
-
- if (cmd.hasOption(APP_PATH)) {
- appPath = cmd.getOptionValue(APP_PATH);
- }
-
- if (cmd.hasOption(EMBEDDED_ACTIVEMQ)) {
- startActiveMq = Boolean.valueOf(cmd.getOptionValue(EMBEDDED_ACTIVEMQ));
- }
-
- if (cmd.hasOption(ACTIVEMQ_BASE)) {
- dataDir = cmd.getOptionValue(ACTIVEMQ_BASE);
- }
-
- if (startActiveMq) {
- BrokerService broker = new BrokerService();
- broker.setUseJmx(false);
- broker.setDataDirectory(dataDir);
- broker.addConnector("vm://localhost");
- broker.addConnector("tcp://localhost:61616");
- broker.start();
- }
-
- EmbeddedServer server = new EmbeddedServer(15000, appPath);
- server.start();
- }
-}