You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sa...@apache.org on 2014/08/04 12:04:03 UTC
[04/27] adding falcon-regression
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml b/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
new file mode 100644
index 0000000..6a0ee2d
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <clusters>
+ <cluster name="corp">
+ <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />
+ </cluster>
+ </clusters>
+ <parallel>2</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(60)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <input end="now(0,0)" start="now(0,-20)"
+ feed="raaw-logs16" name="inputData"/>
+ </inputs>
+ <outputs>
+ <output instance="now(0,0)" feed="agregated-logs16"
+ name="outputData" />
+ </outputs>
+ <properties>
+ <property name="queueName" value="default"/>
+ </properties>
+ <workflow path="/examples/apps/aggregator" />
+ <retry policy="backoff" delay="minutes(10)" attempts="3" />
+
+ <late-process policy="exp-backoff" delay="hours(1)">
+ <late-input feed="impression" workflow-path="hdfs://impression/late/workflow"/>
+ <late-input feed="clicks" workflow-path="hdfs://clicks/late/workflow" />
+ </late-process>
+</process>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <interfaces>
+ <interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+ version="0.20.2" />
+ <interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+ version="0.20.2" />
+ <interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+ <interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+ version="3.1" />
+ <interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+ version="5.1.6" />
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/ivory/staging" />
+ <location name="temp" path="/tmp" />
+ <location name="working" path="/projectsTest/ivory/working" />
+ </locations>
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
new file mode 100644
index 0000000..2114bb9
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <partitions>
+ <partition name="country" />
+ <partition name="colo" />
+ </partitions>
+<!--
+ <groups>online,bi</groups>-->
+
+ <frequency>minutes(20)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
new file mode 100644
index 0000000..08c5f5d
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!-- <partitions>
+ <partition name="fraud" />
+ <partition name="good" />
+ </partitions>
+
+ <groups>online,bi</groups>-->
+
+ <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml b/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
new file mode 100644
index 0000000..3fb4c61
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <clusters>
+ <cluster name="corp">
+ <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />
+ </cluster>
+ </clusters>
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <input end="now(0,0)" start="now(0,-20)"
+ feed="raaw-logs16" name="inputData"/>
+ </inputs>
+ <outputs>
+ <output instance="now(0,0)" feed="agregated-logs16"
+ name="outputData" />
+ </outputs>
+ <properties>
+ <property name="queueName" value="default"/>
+
+ <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+ <property name="user" value="${user()}"/>
+
+ </properties>
+ <workflow path="/examples/apps/aggregator"/>
+ <retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
new file mode 100644
index 0000000..4a80109
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
@@ -0,0 +1,61 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
+ <start to="aggregator"/>
+ <action name="aggregator">
+ <map-reduce>
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${outputData}"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ <property>
+ <name>mapred.mapper.class</name>
+ <value>org.apache.oozie.example.SampleMapper</value>
+ </property>
+ <property>
+ <name>mapred.reducer.class</name>
+ <value>org.apache.oozie.example.SampleReducer</value>
+ </property>
+ <property>
+ <name>mapred.map.tasks</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>mapred.input.dir</name>
+ <value>${inputData}</value>
+ </property>
+ <property>
+ <name>mapred.output.dir</name>
+ <value>${outputData}</value>
+ </property>
+ </configuration>
+ </map-reduce>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+ <kill name="fail">
+ <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <interfaces>
+ <interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+ version="0.20.2" />
+ <interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+ version="0.20.2" />
+ <interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+ <interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+ version="3.1" />
+ <interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+ version="5.1.6" />
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/ivory/staging" />
+ <location name="temp" path="/tmp" />
+ <location name="working" path="/projectsTest/ivory/working" />
+ </locations>
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
new file mode 100644
index 0000000..2da8e1c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <partitions>
+ <partition name="country" />
+ <partition name="colo" />
+ </partitions>
+<!--
+ <groups>online,bi</groups>-->
+
+ <frequency>minutes(20)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
new file mode 100644
index 0000000..1ddac37
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!-- <partitions>
+ <partition name="fraud" />
+ <partition name="good" />
+ </partitions>
+
+ <groups>online,bi</groups>-->
+
+ <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
new file mode 100644
index 0000000..3f23ffe
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <clusters>
+ <cluster name="corp">
+ <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />
+ </cluster>
+ </clusters>
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+
+ </inputs>
+ <outputs>
+ <output instance="now(0,0)" feed="agregated-logs16"
+ name="outputData" />
+ </outputs>
+ <properties>
+ <property name="queueName" value="default"/>
+
+ <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+ <property name="user" value="${user()}"/>
+
+ </properties>
+ <workflow path="/examples/apps/aggregator"/>
+ <retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <interfaces>
+ <interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+ version="0.20.2" />
+ <interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+ version="0.20.2" />
+ <interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+ <interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+ version="3.1" />
+ <interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+ version="5.1.6" />
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/ivory/staging" />
+ <location name="temp" path="/tmp" />
+ <location name="working" path="/projectsTest/ivory/working" />
+ </locations>
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
new file mode 100644
index 0000000..2da8e1c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <partitions>
+ <partition name="country" />
+ <partition name="colo" />
+ </partitions>
+<!--
+ <groups>online,bi</groups>-->
+
+ <frequency>minutes(20)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
new file mode 100644
index 0000000..1ddac37
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!-- <partitions>
+ <partition name="fraud" />
+ <partition name="good" />
+ </partitions>
+
+ <groups>online,bi</groups>-->
+
+ <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+ />
+ <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
new file mode 100644
index 0000000..2175e24
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <clusters>
+ <cluster name="corp">
+ <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />
+ </cluster>
+ </clusters>
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <input end="now(0,0)" start="now(0,-20)"
+ feed="raaw-logs16" name="inputData"/>
+ </inputs>
+ <outputs>
+
+ </outputs>
+ <properties>
+ <property name="queueName" value="default"/>
+
+ <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+ <property name="user" value="${user()}"/>
+
+ </properties>
+ <workflow path="/examples/apps/aggregator"/>
+ <retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
new file mode 100644
index 0000000..319bf5a
Binary files /dev/null and b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
new file mode 100644
index 0000000..6a00752
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<cluster colo="gs" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <interfaces>
+ <interface type="readonly" endpoint="hftp://gs1001.grid.corp.inmobi.com:50070"
+ version="0.20.2" />
+ <interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+ version="0.20.2" />
+ <interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+ <interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+ version="3.1" />
+ <interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+ version="5.1.6" />
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/ivory/staging" />
+ <location name="temp" path="/tmp" />
+ <location name="working" path="/projects/ivory/working" />
+ </locations>
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</cluster>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
new file mode 100644
index 0000000..5350d0a
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!-- <partitions>
+ <partition name="fraud" />
+ <partition name="good" />
+ </partitions>
+
+ <groups>online,bi</groups>-->
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)"/>
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-01-01T00:00Z" end="2099-12-31T23:59Z"/>
+ <retention limit="months(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/lateDataTest/testFolders/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field3" value="value1" />
+ <property name="field4" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
new file mode 100644
index 0000000..d9e4e32
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!-- <partitions>
+ <partition name="fraud" />
+ <partition name="good" />
+ </partitions>
+
+ <groups>online,bi</groups>-->
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(6)" />
+
+ <clusters>
+ <cluster name="corp" type="source">
+ <validity start="2009-01-01T01:00Z" end="2099-12-31T23:59Z"/>
+ <retention limit="months(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/lateDataTest/testFolders/output-data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+ <location type="stats" path="/projects/ivory/clicksStats" />
+ <location type="meta" path="/projects/ivory/clicksMetaData" />
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755" />
+ <schema location="/schema/clicks" provider="protobuf" />
+
+ <properties>
+ <property name="field5" value="value1" />
+ <property name="field6" value="value2" />
+ </properties>
+</feed>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
new file mode 100644
index 0000000..09e8a09
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="corp">
+ <validity end="2010-01-01T01:03Z" start="2010-01-01T01:01Z"/>
+ </cluster>
+ </clusters>
+ <parallel>3</parallel>
+
+ <order>FIFO</order>
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <input end="now(0,0)" start="now(0,-1)" feed="raaw-logs16" name="inputData"/>
+ </inputs>
+ <outputs>
+ <output instance="now(0,0)" feed="agregated-logs16" name="outputData"/>
+ </outputs>
+ <properties>
+ <property name="queueName" value="default"/>
+ </properties>
+ <workflow path="/examples/apps/aggregator"/>
+ <retry policy="periodic" delay="minutes(10)" attempts="3"/>
+
+ <late-process policy="periodic" delay="hours(1)">
+ <late-input input="inputData" workflow-path="hdfs://impression/late/workflow"/>
+ </late-process>
+</process>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
new file mode 100755
index 0000000..d7e0a91
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<feed name="FETL-BillingRC" description="billing RC File" xmlns="uri:falcon:feed:0.1">
+ <partitions>
+ <partition name="colo"/>
+ <partition name="eventTime"/>
+ <partition name="impressionHour"/>
+ <partition name="pricingModel"/>
+ </partitions>
+ <frequency>minutes(5)</frequency>
+ <late-arrival cut-off="days(100000)"/>
+ <clusters>
+ <cluster name="lhr1-emerald" type="target" partition="${cluster.colo}">
+ <validity start="2012-07-20T00:00Z" end="2099-07-16T00:00Z"/>
+ <retention limit="days(10000)" action="delete"/>
+ </cluster>
+ <cluster name="ua2-ruby" type="target" partition="${cluster.colo}">
+ <validity start="2012-07-16T00:00Z" end="2099-07-16T00:00Z"/>
+ <retention limit="days(10000)" action="delete"/>
+ </cluster>
+ <cluster name="prod-gloabl" type="source">
+ <validity start="2012-07-20T03:00Z" end="2099-07-16T00:00Z"/>
+ <retention limit="days(10000)" action="delete"/>
+ </cluster>
+ </clusters>
+ <locations>
+ <location type="data" path="/data/regression/fetlrc/billing/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+ <location type="stats" path="/data/regression/fetlrc/billing/stats"/>
+ <location type="meta" path="/data/regression/fetlrc/billing/metadata"/>
+ </locations>
+ <ACL owner="fetl" group="group" permission="0x755"/>
+ <schema location="/databus/streams_local/click_rr/schema/" provider="protobuf"/>
+
+ <properties>
+ <property name="field1" value="value1" />
+ <property name="field2" value="value2" />
+ </properties>
+</feed>
+
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
new file mode 100755
index 0000000..afa86c4
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<cluster name="ivory2-stg4" description="" colo="stg" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hftp://stg-nn.blue.ua2.inmobi.com:50070" version="0.20.2-cdh3u0"/>
+ <interface type="execute" endpoint="stg-jt.blue.ua2.inmobi.com:54311" version="0.20.2-cdh3u0"/>
+ <interface type="write" endpoint="hdfs://stg-nn.blue.ua2.inmobi.com:54310" version="0.20.2-cdh3u0"/>
+ <interface type="messaging" endpoint="tcp://gs1134.blue.ua2.inmobi.com:61618?daemon=true" version="5.1.6"/>
+ <interface type="workflow" endpoint="http://gs1134.blue.ua2.inmobi.com:11002/oozie/" version="3.1.4"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/ivory2/stg/staging"/>
+ <location name="temp" path="/tmp"/>
+ <location name="working" path="/projects/ivory2/stg/working"/>
+ </locations>
+ <properties>
+ <property name="colo.name" value="ua2"/>
+ <property name="hbase.zookeeper.quorum" value="gs1001.grid.corp.inmobi.com"/>
+ <property name="hbase.zookeeper.property.clientPort" value="2181"/>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
new file mode 100644
index 0000000..1ab0140
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Configuration implements Iterable<Map.Entry<String, String>>, Cloneable {
+
+ private final Map<String, String> properties;
+
+ public Configuration() {
+ properties = new ConcurrentHashMap<String, String>();
+ }
+
+ public Configuration(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public void addConfiguration(Configuration config) {
+ for (Entry<String, String> entry : config) {
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ public Configuration addAndReturnNewConfiguration(Configuration config) {
+ Map<String, String> newProperties = new ConcurrentHashMap<String, String>(properties);
+ for (Entry<String, String> entry : config) {
+ newProperties.put(entry.getKey(), entry.getValue());
+ }
+ return new Configuration(newProperties);
+ }
+
+ public String getConf(String name) {
+ return properties.get(name);
+ }
+
+ public void setConf(String name, String value) {
+ properties.put(name, value);
+ }
+
+ public void setConf(String name, String value, String defaultValue) {
+ if (value == null) {
+ properties.put(name, defaultValue);
+ } else {
+ properties.put(name, value);
+ }
+ }
+
+ @Override
+ public Iterator<Entry<String, String>> iterator() {
+ return properties.entrySet().iterator();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
new file mode 100644
index 0000000..f02ed34
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ivory.entity.common;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DateValidator {
+
+ private Pattern pattern;
+ private Matcher matcher;
+
+ private static final String DATE_PATTERN = "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
+
+ public DateValidator() {
+ pattern = Pattern.compile(DATE_PATTERN);
+ }
+
+ /**
+ * Validate date format with regular expression
+ *
+ * @param date
+ * date address for validation
+ * @return true valid date fromat, false invalid date format
+ */
+ public boolean validate(final String date) {
+
+ matcher = pattern.matcher(date);
+
+ if (matcher.matches()) {
+
+ matcher.reset();
+
+ if (matcher.find()) {
+
+ int year = Integer.parseInt(matcher.group(1));
+ String month = matcher.group(2);
+ String day = matcher.group(3);
+
+ if (day.equals("31")
+ && (month.equals("4") || month.equals("6")
+ || month.equals("9") || month.equals("11")
+ || month.equals("04") || month.equals("06") || month
+ .equals("09"))) {
+ return false; // only 1,3,5,7,8,10,12 has 31 days
+ } else if (month.equals("2") || month.equals("02")) {
+ // leap year
+ if (year % 4 == 0) {
+ if (day.equals("30") || day.equals("31")) {
+ return false;
+ } else {
+ return true;
+ }
+ } else {
+ if (day.equals("29") || day.equals("30")
+ || day.equals("31")) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+ } else {
+ return true;
+ }
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
new file mode 100644
index 0000000..67f9d4c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Calendar;
+
+public enum TimeUnit {
+ MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), MONTH(Calendar.MONTH), END_OF_DAY(Calendar.DATE), END_OF_MONTH(
+ Calendar.MONTH), NONE(-1);
+
+ private int calendarUnit;
+
+ private TimeUnit(int calendarUnit) {
+ this.calendarUnit = calendarUnit;
+ }
+
+ public int getCalendarUnit() {
+ return calendarUnit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
new file mode 100644
index 0000000..3df60b0
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
new file mode 100644
index 0000000..f3f94c5
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[15] LOG!!!!!!
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
new file mode 100644
index 0000000..f992f21
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
@@ -0,0 +1,412 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import org.apache.hadoop.conf.Configuration;
+//import org.apache.hadoop.fs.FileSystem;
+//import org.apache.hadoop.fs.Path;
+//import org.apache.hadoop.io.IOUtils;
+//import org.apache.log4j.Logger;
+//
+//import java.io.ByteArrayOutputStream;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.util.HashMap;
+//import java.util.Map;
+//
+///**
+// * Value Object which is stored in JMS Topic as TextMessage
+// *
+// */
+//public class EntityInstanceMessage {
+//
+// private final Map<ARG, String> msgs = new HashMap<ARG, String>();
+// private static final String MSG_SEPERATOR = "$";
+// private static final int ARG_LENGTH = EntityInstanceMessage.ARG.values().length;
+// private static final Logger LOG = Logger
+// .getLogger(EntityInstanceMessage.class);
+// private static final String IVORY_PROCESS_TOPIC_NAME = "IVORY.PROCESS.TOPIC";
+//
+// /**
+// * Enum for arguments that are used in coordinators to pass arguments to
+// * parent workflow
+// */
+//
+// public enum entityOperation {
+// GENERATE, DELETE, ARCHIVE, REPLICATE, CHMOD
+// }
+//
+// public enum ARG {
+// PROCESS_NAME(0, "entityName"), FEED_NAME(1, "feedNames"), FEED_INSTANCE_PATH(
+// 2, "feedInstancePaths"), WORKFLOW_ID(3, "workflowId"), RUN_ID(
+// 4, "runId"), NOMINAL_TIME(5, "nominalTime"), TIME_STAMP(6,
+// "timeStamp"), BROKER_URL(7, "brokerUrl"), BROKER_IMPL_CLASS(8,
+// "brokerImplClass"), ENTITY_TYPE(9, "entityType"), OPERATION(10,
+// "operation"), LOG_FILE(11, "logFile"), TOPIC_NAME(12,
+// "topicName"), STATUS(13, "status"), BROKER_TTL(14, "broker.ttlInMins");
+//
+// private int argOrder;
+// private String argName;
+//
+// private ARG(int argOrder, String argName) {
+// this.argOrder = argOrder;
+// this.argName = argName;
+// }
+//
+// public int ORDER() {
+// return this.argOrder;
+// }
+//
+// public String NAME() {
+// return this.argName;
+// }
+// }
+//
+// public String getProcessName() {
+// return this.msgs.get(ARG.PROCESS_NAME);
+// }
+//
+// public void setProcessName(String processName) {
+// this.msgs.put(ARG.PROCESS_NAME, processName);
+// }
+//
+// public String getTopicName() {
+// return this.msgs.get(ARG.TOPIC_NAME);
+// }
+//
+// public void setTopicName(String topicName) {
+// this.msgs.put(ARG.TOPIC_NAME, topicName);
+// }
+//
+// public String getFeedName() {
+// return this.msgs.get(ARG.FEED_NAME);
+// }
+//
+// public void setFeedName(String feedName) {
+// this.msgs.put(ARG.FEED_NAME, feedName);
+// }
+//
+// public String getFeedInstancePath() {
+// return this.msgs.get(ARG.FEED_INSTANCE_PATH);
+// }
+//
+// public void setFeedInstancePath(String feedInstancePath) {
+// this.msgs.put(ARG.FEED_INSTANCE_PATH, feedInstancePath);
+// }
+//
+// public String getWorkflowId() {
+// return this.msgs.get(ARG.WORKFLOW_ID);
+// }
+//
+// public void setWorkflowId(String workflowId) {
+// this.msgs.put(ARG.WORKFLOW_ID, workflowId);
+// }
+//
+// public String getRunId() {
+// return this.msgs.get(ARG.RUN_ID);
+// }
+//
+// public void setRunId(String runId) {
+// this.msgs.put(ARG.RUN_ID, runId);
+// }
+//
+// public String getNominalTime() {
+// return this.msgs.get(ARG.NOMINAL_TIME);
+// }
+//
+// public void setNominalTime(String nominalTime) {
+// this.msgs.put(ARG.NOMINAL_TIME, nominalTime);
+// }
+//
+// public String getTimeStamp() {
+// return this.msgs.get(ARG.TIME_STAMP);
+// }
+//
+// public void setTimeStamp(String timeStamp) {
+// this.msgs.put(ARG.TIME_STAMP, timeStamp);
+// }
+//
+// public String getBrokerUrl() {
+// return this.msgs.get(ARG.BROKER_URL);
+// }
+//
+// public void setBrokerUrl(String brokerUrl) {
+// this.msgs.put(ARG.BROKER_URL, brokerUrl);
+// }
+//
+// public String getBrokerImplClass() {
+// return this.msgs.get(ARG.BROKER_IMPL_CLASS);
+// }
+//
+// public void setBrokerImplClass(String brokerImplClass) {
+// this.msgs.put(ARG.BROKER_IMPL_CLASS, brokerImplClass);
+// }
+//
+// public String getEntityType() {
+// return this.msgs.get(ARG.ENTITY_TYPE);
+// }
+//
+// public void setEntityType(String entityType) {
+// this.msgs.put(ARG.ENTITY_TYPE, entityType);
+// }
+//
+// public String getOperation() {
+// return this.msgs.get(ARG.OPERATION);
+// }
+//
+// public void setOperation(String operation) {
+// this.msgs.put(ARG.OPERATION, operation);
+// }
+//
+// public String getLogFile() {
+// return this.msgs.get(ARG.LOG_FILE);
+// }
+//
+// public void setLogFile(String logFile) {
+// this.msgs.put(ARG.LOG_FILE, logFile);
+// }
+//
+// public String getStatus() {
+// return this.msgs.get(ARG.STATUS);
+// }
+//
+// public void setStatus(String status) {
+// this.msgs.put(ARG.STATUS, status);
+// }
+//
+// public String getBrokerTTL() {
+// return this.msgs.get(ARG.BROKER_TTL);
+// }
+//
+// public void setBrokerTTL(String brokerTTL) {
+// this.msgs.put(ARG.BROKER_TTL, brokerTTL);
+// }
+//
+// @Override
+// public String toString() {
+// if (getEntityType().equalsIgnoreCase("PROCESS")
+// && getTopicName().equals(IVORY_PROCESS_TOPIC_NAME)) {
+// return getIvoryMessage();
+// }
+// if (getEntityType().equalsIgnoreCase("FEED")) {
+// return getFeedMessage();
+// }
+// return getProcessMessage();
+//
+// }
+//
+// private String getProcessMessage() {
+// return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+// + getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+// + MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+// + MSG_SEPERATOR + getTimeStamp();
+// }
+//
+// private String getIvoryMessage() {
+// return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+// + getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+// + MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+// + MSG_SEPERATOR + getTimeStamp() + MSG_SEPERATOR + getStatus();
+// }
+//
+// private String getFeedMessage() {
+// return getFeedName() + MSG_SEPERATOR
+// + getFeedInstancePath() + MSG_SEPERATOR + getOperation()
+// + MSG_SEPERATOR + getWorkflowId() + MSG_SEPERATOR + getRunId()
+// + MSG_SEPERATOR + getNominalTime() + MSG_SEPERATOR
+// + getTimeStamp();
+// }
+//
+// /**
+// *
+// * @param args
+// * - String array passed from oozie action for jms messaging
+// * @return ProcessMessage - Value object which is stored in JMS topic
+// */
+// public static EntityInstanceMessage[] argsToMessage(String[] args) {
+//
+// assert args.length == ARG_LENGTH : "Required number of arguments: "
+// + ARG_LENGTH;
+//
+// String[] feedNames = getFeedNames(args);
+//
+// String[] feedPaths;
+// try {
+// feedPaths = getFeedPaths(args);
+// } catch (IOException e) {
+// LOG.error("Error getting instance paths: ", e);
+// throw new RuntimeException(e);
+// }
+//
+// EntityInstanceMessage[] processMessages = new EntityInstanceMessage[feedPaths.length];
+// for (int i = 0; i < feedPaths.length; i++) {
+// EntityInstanceMessage instanceMessage = new EntityInstanceMessage();
+// instanceMessage
+// .setProcessName(args[EntityInstanceMessage.ARG.PROCESS_NAME
+// .ORDER()]);
+// if (args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()]
+// .equalsIgnoreCase("PROCESS")) {
+// instanceMessage.setFeedName(feedNames[i]);
+// } else {
+// instanceMessage
+// .setFeedName(args[EntityInstanceMessage.ARG.FEED_NAME
+// .ORDER()]);
+// }
+// instanceMessage.setFeedInstancePath(feedPaths[i]);
+// instanceMessage
+// .setWorkflowId(args[EntityInstanceMessage.ARG.WORKFLOW_ID
+// .ORDER()]);
+// instanceMessage.setRunId(args[EntityInstanceMessage.ARG.RUN_ID
+// .ORDER()]);
+// instanceMessage
+// .setNominalTime(args[EntityInstanceMessage.ARG.NOMINAL_TIME
+// .ORDER()]);
+// instanceMessage
+// .setTimeStamp(args[EntityInstanceMessage.ARG.TIME_STAMP
+// .ORDER()]);
+// instanceMessage
+// .setBrokerUrl(args[EntityInstanceMessage.ARG.BROKER_URL
+// .ORDER()]);
+// instanceMessage
+// .setBrokerImplClass(args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS
+// .ORDER()]);
+// instanceMessage
+// .setEntityType(args[EntityInstanceMessage.ARG.ENTITY_TYPE
+// .ORDER()]);
+// instanceMessage
+// .setOperation(args[EntityInstanceMessage.ARG.OPERATION
+// .ORDER()]);
+// instanceMessage.setLogFile(args[EntityInstanceMessage.ARG.LOG_FILE
+// .ORDER()]);
+// instanceMessage
+// .setTopicName(args[EntityInstanceMessage.ARG.TOPIC_NAME
+// .ORDER()]);
+// instanceMessage.setStatus(args[EntityInstanceMessage.ARG.STATUS
+// .ORDER()]);
+// instanceMessage.setBrokerTTL(args[EntityInstanceMessage.ARG.BROKER_TTL
+// .ORDER()]);
+//
+// processMessages[i] = instanceMessage;
+// }
+// return processMessages;
+// }
+//
+// private static String[] getFeedNames(String[] args) {
+// String topicName = args[ARG.TOPIC_NAME.argOrder];
+// if (topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+// return new String[] { args[EntityInstanceMessage.ARG.FEED_NAME
+// .ORDER()] };
+// }
+// return args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()].split(",");
+// }
+//
+// private static String[] getFeedPaths(String[] args) throws IOException {
+// String entityType = args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()];
+// String topicName = args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()];
+//
+// if (entityType.equalsIgnoreCase("PROCESS")
+// && topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+// LOG.debug("Returning instance paths for Ivory Topic: "
+// + args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+// return new String[] { args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH
+// .ORDER()] };
+// }
+//
+// if (entityType.equalsIgnoreCase("PROCESS")) {
+// LOG.debug("Returning instance paths for process: "
+// + args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+// return args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]
+// .split(",");
+// }
+// //
+// Path logFile = new Path(
+// args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()]);
+// FileSystem fs = FileSystem.get(logFile.toUri(), new Configuration());
+// ByteArrayOutputStream writer = new ByteArrayOutputStream();
+// InputStream instance = fs.open(logFile);
+// IOUtils.copyBytes(instance, writer, 4096, true);
+// String[] instancePaths = writer.toString().split("=");
+// if (instancePaths.length == 1) {
+// LOG.debug("Returning 0 instance paths for feed ");
+// return new String[0];
+// } else {
+// LOG.debug("Returning instance paths for feed " + instancePaths[1]);
+// return instancePaths[1].split(",");
+// }
+//
+// }
+//
+// /**
+// *
+// * @param instanceMessages
+// * - value object which is stored in JMS topic as TextMessage
+// * @return - String array.
+// */
+// public static String[] messageToArgs(
+// EntityInstanceMessage[] instanceMessages) {
+// String[] args = new String[ARG_LENGTH];
+//
+// args[EntityInstanceMessage.ARG.PROCESS_NAME.ORDER()] = instanceMessages[0]
+// .getProcessName();
+// StringBuilder feedNames = new StringBuilder();
+// StringBuilder feedPaths = new StringBuilder();
+//
+// for (EntityInstanceMessage instanceMessage : instanceMessages) {
+// feedNames.append(instanceMessage.getFeedName()).append(",");
+// feedPaths.append(instanceMessage.getFeedInstancePath()).append(",");
+// }
+// if (instanceMessages[0].getEntityType().equalsIgnoreCase("PROCESS")) {
+// args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = feedNames
+// .toString();
+//
+// } else {
+// args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = instanceMessages[0]
+// .getFeedName();
+// }
+// args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()] = feedPaths
+// .toString();
+// args[EntityInstanceMessage.ARG.WORKFLOW_ID.ORDER()] = instanceMessages[0]
+// .getWorkflowId();
+// args[EntityInstanceMessage.ARG.RUN_ID.ORDER()] = instanceMessages[0]
+// .getRunId();
+// args[EntityInstanceMessage.ARG.NOMINAL_TIME.ORDER()] = instanceMessages[0]
+// .getNominalTime();
+// args[EntityInstanceMessage.ARG.TIME_STAMP.ORDER()] = instanceMessages[0]
+// .getTimeStamp();
+// args[EntityInstanceMessage.ARG.BROKER_URL.ORDER()] = instanceMessages[0]
+// .getBrokerUrl();
+// args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()] = instanceMessages[0]
+// .getBrokerImplClass();
+// args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()] = instanceMessages[0]
+// .getEntityType();
+// args[EntityInstanceMessage.ARG.OPERATION.ORDER()] = instanceMessages[0]
+// .getOperation();
+// args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()] = instanceMessages[0]
+// .getLogFile();
+// args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()] = instanceMessages[0]
+// .getTopicName();
+// args[EntityInstanceMessage.ARG.STATUS.ORDER()] = instanceMessages[0]
+// .getStatus();
+// args[EntityInstanceMessage.ARG.BROKER_TTL.ORDER()] = instanceMessages[0]
+// .getBrokerTTL();
+//
+// return args;
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
new file mode 100644
index 0000000..f7e8bbb
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
@@ -0,0 +1,61 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.TextMessage;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Ivory JMS message creator- creates JMS TextMessage
+// */
+//public class EntityInstanceMessageCreator {
+//
+// private static final Logger LOG = Logger
+// .getLogger(EntityInstanceMessageCreator.class);
+//
+// private TextMessage textMessage;
+//
+// private final EntityInstanceMessage args;
+//
+// public EntityInstanceMessageCreator(EntityInstanceMessage args) {
+// this.args = args;
+// }
+//
+// public TextMessage createMessage(Session session) throws JMSException {
+// this.textMessage = session.createTextMessage();
+// this.textMessage.setText(this.args.toString());
+// LOG.debug("Sending Message: " + this.textMessage.getText());
+// // System.out.println("Sending Message: " + this.textMessage);
+// return this.textMessage;
+// }
+//
+// @Override
+// public String toString() {
+// try {
+// return this.textMessage.getText();
+// } catch (JMSException e) {
+// return e.getMessage();
+// }
+//
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
new file mode 100644
index 0000000..7234f52
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
@@ -0,0 +1,139 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import java.lang.reflect.InvocationTargetException;
+//
+//import javax.jms.Connection;
+//import javax.jms.ConnectionFactory;
+//import javax.jms.DeliveryMode;
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.Topic;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Default Ivory Message Producer The configuration are loaded from
+// * jms-beans.xml
+// */
+//public class MessageProducer {
+//
+// private Connection connection;
+// private static final Logger LOG = Logger.getLogger(MessageProducer.class);
+// private static final long DEFAULT_TTL = 3 * 24 * 60 * 60 * 1000;
+//
+// /**
+// *
+// * @param arguments
+// * - Accepts a Message to be send to JMS topic, creates a new
+// * Topic based on topic name if it does not exist or else
+// * existing topic with the same name is used to send the message.
+// * @throws JMSException
+// */
+// protected void sendMessage(EntityInstanceMessage entityInstanceMessage)
+// throws JMSException {
+//
+// Session session = connection.createSession(false,
+// Session.AUTO_ACKNOWLEDGE);
+// Topic entityTopic = session.createTopic(entityInstanceMessage
+// .getTopicName());
+// javax.jms.MessageProducer producer = session
+// .createProducer(entityTopic);
+// producer.setDeliveryMode(DeliveryMode.PERSISTENT);
+// long messageTTL = DEFAULT_TTL;
+// try {
+// long messageTTLinMins = Long.parseLong(entityInstanceMessage
+// .getBrokerTTL());
+// messageTTL = messageTTLinMins * 60 * 1000;
+// } catch (NumberFormatException e) {
+// LOG.error("Error in parsing broker.ttl, setting TTL to:"
+// + DEFAULT_TTL+ " milli-seconds");
+// }
+// producer.setTimeToLive(messageTTL);
+// producer.send(new EntityInstanceMessageCreator(entityInstanceMessage)
+// .createMessage(session));
+//
+// }
+//
+// /**
+// *
+// * @param args
+// * - array of Strings, which will be used to create TextMessage
+// */
+// public static void main(String[] args) {
+// debug(args);
+// EntityInstanceMessage[] entityInstanceMessage = EntityInstanceMessage
+// .argsToMessage(args);
+// if (entityInstanceMessage.length == 0) {
+// LOG.warn("No operation on output feed");
+// return;
+// }
+//
+// MessageProducer ivoryMessageProducer = new MessageProducer();
+// try {
+// ivoryMessageProducer.createAndStartConnection(
+// args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()],
+// "", "", entityInstanceMessage[0].getBrokerUrl());
+// for (EntityInstanceMessage processMessage : entityInstanceMessage) {
+// ivoryMessageProducer.sendMessage(processMessage);
+// }
+// } catch (JMSException e) {
+// LOG.error(e);
+// e.printStackTrace();
+// } catch (Exception e) {
+// LOG.error(e);
+// e.printStackTrace();
+// } finally {
+// try {
+// ivoryMessageProducer.connection.close();
+// } catch (JMSException e) {
+// e.printStackTrace();
+// }
+// }
+//
+// }
+//
+// private static void debug(String[] args) {
+// if (LOG.isDebugEnabled()) {
+// for (int i = 0; i < args.length; i++) {
+// LOG.debug(args[i] + "::");
+// }
+// }
+//
+// }
+//
+// private void createAndStartConnection(String implementation,
+// String userName, String password, String url) throws JMSException,
+// ClassNotFoundException, IllegalArgumentException,
+// SecurityException, InstantiationException, IllegalAccessException,
+// InvocationTargetException, NoSuchMethodException {
+//
+// Class<ConnectionFactory> clazz = (Class<ConnectionFactory>) MessageProducer.class
+// .getClassLoader().loadClass(implementation);
+//
+// ConnectionFactory connectionFactory = clazz.getConstructor(
+// String.class, String.class, String.class).newInstance(userName,
+// password, url);
+//
+// connection = connectionFactory.createConnection();
+// connection.start();
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
new file mode 100644
index 0000000..4c61a15
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+test file needed
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
new file mode 100644
index 0000000..4d3d972
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
@@ -0,0 +1,31 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+
+#Global configuration for Ivory JMS, default to use embedded ActiveMQ.
+
+#Defualt Active MQ url
+#ivory.broker.url = tcp://localhost:61616?daemon=true
+
+#Embedded
+ivory.broker.url = vm://localhost?broker.useJmx=false&broker.persistent=true
+
+
+