You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pe...@apache.org on 2016/04/23 06:54:05 UTC
[1/2] falcon git commit: FALCON-1899 Create examples artifact module
in Falcon
Repository: falcon
Updated Branches:
refs/heads/master b49056d6c -> 2945fa80b
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-pig-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-pig-process.xml b/src/main/examples/entity/hcat/hcat-pig-process.xml
deleted file mode 100644
index 60836c4..0000000
--- a/src/main/examples/entity/hcat/hcat-pig-process.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<process name="hcat-pig-process" xmlns="uri:falcon:process:0.1">
- <clusters>
- <cluster name="hcat-local">
- <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
- </cluster>
- </clusters>
-
- <parallel>1</parallel>
- <order>FIFO</order>
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
-
- <inputs>
- <!-- In the pig script, the input paths will be available in a variable 'inparts' -->
- <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/>
- </inputs>
-
- <outputs>
- <!-- In the pig script, the output path will be available in a variable 'outparts' -->
- <output name="outpart" feed="hcat-out" instance="now(0,0)"/>
- </outputs>
-
- <workflow engine="pig" path="/app/pig/hcat-wordcount.pig"/>
-</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-replication-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-replication-feed.xml b/src/main/examples/entity/hcat/hcat-replication-feed.xml
deleted file mode 100644
index 2b8f026..0000000
--- a/src/main/examples/entity/hcat/hcat-replication-feed.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="input" name="hcat-repl-feed" xmlns="uri:falcon:feed:0.1">
- <groups>input</groups>
-
- <frequency>minutes(1)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="hcat-local" type="source">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- <cluster name="hcat-local-target" type="target">
- <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- <table uri="catalog:default:repl_in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
- </cluster>
- </clusters>
-
- <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/log/log.format.csv" provider="csv"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-standalone-cluster.xml b/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
deleted file mode 100644
index 7962b1f..0000000
--- a/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!--
- Example cluster definition that can be used with falcon and standalone
- hadoop and oozie instances
--->
-<cluster colo="local" description="Standalone cluster with hacatalog" name="hcat-local" xmlns="uri:falcon:cluster:0.1">
- <interfaces>
- <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
-
- <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
-
- <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
-
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
-
- <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/hcat-staging"/>
- <location name="temp" path="/projects/falcon/hcat-tmp"/>
- <location name="working" path="/projects/falcon/hcat-working"/>
- </locations>
- <properties>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml b/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
deleted file mode 100644
index 8de78cf..0000000
--- a/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!--
- Example cluster definition that can be used with falcon and standalone
- hadoop and oozie instances
--->
-<cluster colo="hcat-local-target" description="Standalone cluster with hacatalog" name="hcat-local-target" xmlns="uri:falcon:cluster:0.1">
- <interfaces>
- <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
-
- <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
-
- <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
-
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
-
- <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/hcat-staging-target"/>
- <location name="temp" path="/projects/falcon/hcat-tmp-target"/>
- <location name="working" path="/projects/falcon/hcat-working-target"/>
- </locations>
- <properties>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hive-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hive-process.xml b/src/main/examples/entity/hcat/hive-process.xml
deleted file mode 100644
index ad8b219..0000000
--- a/src/main/examples/entity/hcat/hive-process.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<process name="hive-process" xmlns="uri:falcon:process:0.1">
- <clusters>
- <cluster name="hcat-local">
- <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
- </cluster>
- </clusters>
-
- <parallel>1</parallel>
- <order>FIFO</order>
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
-
- <inputs>
- <!-- In the pig script, the input paths will be available in a variable 'inparts' -->
- <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/>
- </inputs>
-
- <outputs>
- <!-- In the pig script, the output path will be available in a variable 'outpart' -->
- <output name="outpart" feed="hcat-out" instance="now(0,0)"/>
- </outputs>
-
- <workflow engine="hive" path="/app/hive/wordcount.hql"/>
-</process>
[2/2] falcon git commit: FALCON-1899 Create examples artifact module
in Falcon
Posted by pe...@apache.org.
FALCON-1899 Create examples artifact module in Falcon
Author: peeyush b <pb...@hortonworks.com>
Reviewers: Balu <bv...@hortonworks.com>, Venkatesan <me...@gmail.com>, Venkat Ranganathan <n....@live.com>
Closes #100 from peeyushb/FALCON-1899 and squashes the following commits:
0824204 [peeyush b] changed examples path in assembly-standalone.xml
cdcb3cd [peeyush b] removed extra line
26bc4f8 [peeyush b] FALCON-1899 : Create examples artifact module in Falcon
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/2945fa80
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/2945fa80
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/2945fa80
Branch: refs/heads/master
Commit: 2945fa80b9a46b003d64d2cf2dbe10232e4b4e55
Parents: b49056d
Author: Peeyush Bishnoi <pe...@apache.org>
Authored: Sat Apr 23 10:23:44 2016 +0530
Committer: peeyush b <pb...@hortonworks.com>
Committed: Sat Apr 23 10:23:44 2016 +0530
----------------------------------------------------------------------
examples/app/hive/wordcount.hql | 19 ++++++
examples/app/oozie-mr/workflow.xml | 63 ++++++++++++++++++++
examples/app/pig/hcat-wordcount.pig | 23 +++++++
examples/app/pig/wordcount.pig | 23 +++++++
examples/data/generate.sh | 53 ++++++++++++++++
examples/data/hcat-generate.sh | 45 ++++++++++++++
examples/entity/filesystem/embedded-cluster.xml | 51 ++++++++++++++++
examples/entity/filesystem/in-feed.xml | 39 ++++++++++++
examples/entity/filesystem/oozie-mr-process.xml | 50 ++++++++++++++++
examples/entity/filesystem/out-feed.xml | 39 ++++++++++++
examples/entity/filesystem/pig-process.xml | 43 +++++++++++++
examples/entity/filesystem/replication-feed.xml | 46 ++++++++++++++
.../entity/filesystem/standalone-cluster.xml | 43 +++++++++++++
.../filesystem/standalone-target-cluster.xml | 43 +++++++++++++
examples/entity/hcat/hcat-in-feed.xml | 37 ++++++++++++
examples/entity/hcat/hcat-out-feed.xml | 35 +++++++++++
examples/entity/hcat/hcat-pig-process.xml | 43 +++++++++++++
examples/entity/hcat/hcat-replication-feed.xml | 42 +++++++++++++
.../entity/hcat/hcat-standalone-cluster.xml | 45 ++++++++++++++
.../hcat/hcat-standalone-target-cluster.xml | 45 ++++++++++++++
examples/entity/hcat/hive-process.xml | 43 +++++++++++++
examples/pom.xml | 47 +++++++++++++++
pom.xml | 1 +
src/main/assemblies/assembly-standalone.xml | 4 +-
src/main/assemblies/distributed-package.xml | 17 +++++-
src/main/assemblies/standalone-package.xml | 16 ++++-
src/main/examples/app/hive/wordcount.hql | 19 ------
src/main/examples/app/oozie-mr/workflow.xml | 63 --------------------
src/main/examples/app/pig/hcat-wordcount.pig | 23 -------
src/main/examples/app/pig/wordcount.pig | 23 -------
src/main/examples/data/generate.sh | 53 ----------------
src/main/examples/data/hcat-generate.sh | 45 --------------
.../entity/filesystem/embedded-cluster.xml | 51 ----------------
src/main/examples/entity/filesystem/in-feed.xml | 39 ------------
.../entity/filesystem/oozie-mr-process.xml | 50 ----------------
.../examples/entity/filesystem/out-feed.xml | 39 ------------
.../examples/entity/filesystem/pig-process.xml | 43 -------------
.../entity/filesystem/replication-feed.xml | 46 --------------
.../entity/filesystem/standalone-cluster.xml | 43 -------------
.../filesystem/standalone-target-cluster.xml | 43 -------------
src/main/examples/entity/hcat/hcat-in-feed.xml | 37 ------------
src/main/examples/entity/hcat/hcat-out-feed.xml | 35 -----------
.../examples/entity/hcat/hcat-pig-process.xml | 43 -------------
.../entity/hcat/hcat-replication-feed.xml | 42 -------------
.../entity/hcat/hcat-standalone-cluster.xml | 45 --------------
.../hcat/hcat-standalone-target-cluster.xml | 45 --------------
src/main/examples/entity/hcat/hive-process.xml | 43 -------------
47 files changed, 949 insertions(+), 876 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/hive/wordcount.hql
----------------------------------------------------------------------
diff --git a/examples/app/hive/wordcount.hql b/examples/app/hive/wordcount.hql
new file mode 100644
index 0000000..4c101c5
--- /dev/null
+++ b/examples/app/hive/wordcount.hql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+FROM ${falcon_inparts_table} INSERT OVERWRITE TABLE ${falcon_outpart_table} PARTITION(${falcon_outpart_partitions}) SELECT word, SUM(cnt) as cnt WHERE ${falcon_inparts_filter} GROUP BY word;
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/oozie-mr/workflow.xml
----------------------------------------------------------------------
diff --git a/examples/app/oozie-mr/workflow.xml b/examples/app/oozie-mr/workflow.xml
new file mode 100644
index 0000000..c715c91
--- /dev/null
+++ b/examples/app/oozie-mr/workflow.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf">
+ <start to="mr-node"/>
+ <action name="mr-node">
+ <map-reduce>
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${outpath}"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ <property>
+ <name>mapred.mapper.class</name>
+ <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
+ </property>
+ <property>
+ <name>mapred.reducer.class</name>
+ <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
+ </property>
+ <property>
+ <name>mapred.map.tasks</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>mapred.input.dir</name>
+ <value>${inpaths}</value>
+ </property>
+ <property>
+ <name>mapred.output.dir</name>
+ <value>${outpath}</value>
+ </property>
+ </configuration>
+ </map-reduce>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+ <kill name="fail">
+ <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/pig/hcat-wordcount.pig
----------------------------------------------------------------------
diff --git a/examples/app/pig/hcat-wordcount.pig b/examples/app/pig/hcat-wordcount.pig
new file mode 100644
index 0000000..3df93c0
--- /dev/null
+++ b/examples/app/pig/hcat-wordcount.pig
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+indata = LOAD '$falcon_inparts_table' USING org.apache.hive.hcatalog.pig.HCatLoader();
+filterdata = FILTER indata BY $falcon_inparts_filter;
+grpdata = GROUP filterdata BY (word);
+finaldata = FOREACH grpdata GENERATE FLATTEN(group) as word, (int)SUM(filterdata.cnt) as cnt;
+STORE finaldata INTO '$falcon_outpart_table' USING org.apache.hive.hcatalog.pig.HCatStorer('$falcon_outpart_partitions');
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/pig/wordcount.pig
----------------------------------------------------------------------
diff --git a/examples/app/pig/wordcount.pig b/examples/app/pig/wordcount.pig
new file mode 100644
index 0000000..190d2d4
--- /dev/null
+++ b/examples/app/pig/wordcount.pig
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+indata = LOAD '$inpaths' USING PigStorage() AS (words:chararray);
+tokdata = FOREACH indata GENERATE FLATTEN(TOKENIZE(words)) as word;
+grpdata = GROUP tokdata BY (word);
+finaldata = FOREACH grpdata GENERATE FLATTEN(group), COUNT(tokdata);
+STORE finaldata INTO '$outpath' USING PigStorage('\t');
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/data/generate.sh
----------------------------------------------------------------------
diff --git a/examples/data/generate.sh b/examples/data/generate.sh
new file mode 100644
index 0000000..54db3d7
--- /dev/null
+++ b/examples/data/generate.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR};pwd`
+
+rm -rf generated-data
+YEAR=`date +%Y`
+MONTH=`date +m`
+DAY=`date +%d`
+HOUR=`date +%H`
+
+DELIM='\t'
+input=(first second third fourth fifth)
+for MINUTE in `seq -w 00 59`
+do
+ mkdir -p generated-data/00/$MINUTE/
+ word=${input[$RANDOM % 5]}
+ cnt=`expr $RANDOM % 10`
+ echo -e "$word$DELIM$cnt" > generated-data/00/$MINUTE/data
+done
+
+hadoop fs -rmr /data/in/2013/11/15/
+hadoop fs -mkdir -p /data/in/2013/11/15/
+hadoop fs -put generated-data/00 /data/in/2013/11/15/
+rm -rf generated-data
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/data/hcat-generate.sh
----------------------------------------------------------------------
diff --git a/examples/data/hcat-generate.sh b/examples/data/hcat-generate.sh
new file mode 100644
index 0000000..5c4b40a
--- /dev/null
+++ b/examples/data/hcat-generate.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR};pwd`
+
+${BASEDIR}/generate.sh
+
+hcat -e "DROP TABLE IF EXISTS in_table"
+hcat -e "DROP TABLE IF EXISTS repl_in_table"
+hcat -e "DROP TABLE IF EXISTS out_table"
+hcat -e "CREATE TABLE in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
+hcat -e "CREATE TABLE repl_in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
+hcat -e "CREATE TABLE out_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
+for MINUTE in `seq -w 00 59`
+do
+ hcat -e "ALTER TABLE in_table ADD PARTITION (ds='2013-11-15-00-$MINUTE') LOCATION '/data/in/2013/11/15/00/$MINUTE';"
+done
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/embedded-cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/embedded-cluster.xml b/examples/entity/filesystem/embedded-cluster.xml
new file mode 100644
index 0000000..c505066
--- /dev/null
+++ b/examples/entity/filesystem/embedded-cluster.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ Example cluster definition that can be used with the embedded
+ version of the falcon system started through mvn jetty:run
+
+ How to bring up the test falcon instance:
+
+ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
+ rm -rf target/store/*
+ rm -rf webapp/target/webapps/oozie/data/*
+ cd webapp
+ mvn jetty:run
+-->
+<cluster colo="local" description="" name="local" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hftp://localhost:41110" version="1.1.2"/>
+
+ <interface type="write" endpoint="hdfs://localhost:41020" version="1.1.2"/>
+
+ <interface type="execute" endpoint="localhost:41021" version="1.1.2"/>
+
+ <interface type="workflow" endpoint="http://localhost:41000/oozie/" version="4.0.0"/>
+
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/falcon/staging"/> <!--mandatory-->
+ <location name="temp" path="/projects/falcon/tmp"/> <!--optional-->
+ <location name="working" path="/projects/falcon/working"/> <!--optional-->
+ </locations>
+ <properties>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/in-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/in-feed.xml b/examples/entity/filesystem/in-feed.xml
new file mode 100644
index 0000000..798de9c
--- /dev/null
+++ b/examples/entity/filesystem/in-feed.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="input" name="in" xmlns="uri:falcon:feed:0.1">
+ <groups>input</groups>
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="local">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+ </locations>
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/oozie-mr-process.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/oozie-mr-process.xml b/examples/entity/filesystem/oozie-mr-process.xml
new file mode 100644
index 0000000..c080436
--- /dev/null
+++ b/examples/entity/filesystem/oozie-mr-process.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="oozie-mr-process" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="local">
+ <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
+ </cluster>
+ </clusters>
+
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <!-- In the workflow, the input paths will be available in a variable 'inpaths' -->
+ <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/>
+ </inputs>
+
+ <outputs>
+ <!-- In the workflow, the output path will be available in a variable 'outpath' -->
+ <output name="outpath" feed="out" instance="now(0,0)"/>
+ </outputs>
+
+ <properties>
+ <!-- In the workflow, these properties will be available with variable - key -->
+ <property name="queueName" value="default"/>
+ <!-- The schedule time available as a property in workflow -->
+ <property name="time" value="${instanceTime()}"/>
+ </properties>
+
+ <workflow engine="oozie" path="/app/oozie-mr"/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/out-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/out-feed.xml b/examples/entity/filesystem/out-feed.xml
new file mode 100644
index 0000000..f537bbc
--- /dev/null
+++ b/examples/entity/filesystem/out-feed.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="output" name="out" xmlns="uri:falcon:feed:0.1">
+ <groups>output</groups>
+
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="local">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/data/out/${YEAR}/${MONTH}/${DAY}/${HOUR}"/>
+ </locations>
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/out/out.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/pig-process.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/pig-process.xml b/examples/entity/filesystem/pig-process.xml
new file mode 100644
index 0000000..99e03bd
--- /dev/null
+++ b/examples/entity/filesystem/pig-process.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="pig-process" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="local">
+ <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
+ </cluster>
+ </clusters>
+
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <!-- In the pig script, the input paths will be available in a variable 'inpaths' -->
+ <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/>
+ </inputs>
+
+ <outputs>
+ <!-- In the pig script, the output path will be available in a variable 'outpath' -->
+ <output name="outpath" feed="out" instance="now(0,0)"/>
+ </outputs>
+
+ <workflow engine="pig" path="/app/pig/wordcount.pig"/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/replication-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/replication-feed.xml b/examples/entity/filesystem/replication-feed.xml
new file mode 100644
index 0000000..7387d45
--- /dev/null
+++ b/examples/entity/filesystem/replication-feed.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="replication feed" name="repl-feed" xmlns="uri:falcon:feed:0.1">
+ <groups>input</groups>
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="local" type="source">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ <cluster name="local-target" type="target">
+ <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ <locations>
+ <location type="data" path="/data/repl-in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+ </locations>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+ </locations>
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/standalone-cluster.xml b/examples/entity/filesystem/standalone-cluster.xml
new file mode 100644
index 0000000..4f8a5fc
--- /dev/null
+++ b/examples/entity/filesystem/standalone-cluster.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ Example cluster definition that can be used with falcon and standalone
+ hadoop and oozie instances
+-->
+<cluster colo="local" description="Standalone cluster" name="local" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+ <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/falcon/staging"/> <!--mandatory-->
+ <location name="temp" path="/projects/falcon/tmp"/> <!--optional-->
+ <location name="working" path="/projects/falcon/working"/> <!--optional-->
+ </locations>
+ <properties>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/filesystem/standalone-target-cluster.xml b/examples/entity/filesystem/standalone-target-cluster.xml
new file mode 100644
index 0000000..cb5c4c6
--- /dev/null
+++ b/examples/entity/filesystem/standalone-target-cluster.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ Example cluster definition that can be used with falcon and standalone
+ hadoop and oozie instances
+-->
+<cluster colo="local" description="Standalone cluster" name="local-target" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+ <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/falcon/staging-target"/> <!--mandatory-->
+ <location name="temp" path="/projects/falcon/tmp-target"/> <!--optional-->
+ <location name="working" path="/projects/falcon/working-target"/> <!--optional-->
+ </locations>
+ <properties>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-in-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-in-feed.xml b/examples/entity/hcat/hcat-in-feed.xml
new file mode 100644
index 0000000..f978e8c
--- /dev/null
+++ b/examples/entity/hcat/hcat-in-feed.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="input" name="hcat-in" xmlns="uri:falcon:feed:0.1">
+ <groups>input</groups>
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="hcat-local">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ </clusters>
+
+ <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-out-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-out-feed.xml b/examples/entity/hcat/hcat-out-feed.xml
new file mode 100644
index 0000000..23b22b9
--- /dev/null
+++ b/examples/entity/hcat/hcat-out-feed.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="output" name="hcat-out" xmlns="uri:falcon:feed:0.1">
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="hcat-local">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ </clusters>
+
+ <table uri="catalog:default:out_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/out/out.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-pig-process.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-pig-process.xml b/examples/entity/hcat/hcat-pig-process.xml
new file mode 100644
index 0000000..60836c4
--- /dev/null
+++ b/examples/entity/hcat/hcat-pig-process.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="hcat-pig-process" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="hcat-local">
+ <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
+ </cluster>
+ </clusters>
+
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <!-- In the pig script, the input paths will be available in a variable 'inparts' -->
+ <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/>
+ </inputs>
+
+ <outputs>
+ <!-- In the pig script, the output path will be available in a variable 'outparts' -->
+ <output name="outpart" feed="hcat-out" instance="now(0,0)"/>
+ </outputs>
+
+ <workflow engine="pig" path="/app/pig/hcat-wordcount.pig"/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-replication-feed.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-replication-feed.xml b/examples/entity/hcat/hcat-replication-feed.xml
new file mode 100644
index 0000000..2b8f026
--- /dev/null
+++ b/examples/entity/hcat/hcat-replication-feed.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="input" name="hcat-repl-feed" xmlns="uri:falcon:feed:0.1">
+ <groups>input</groups>
+
+ <frequency>minutes(1)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="hours(1)"/>
+
+ <clusters>
+ <cluster name="hcat-local" type="source">
+ <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ </cluster>
+ <cluster name="hcat-local-target" type="target">
+ <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
+ <retention limit="hours(2)" action="delete"/>
+ <table uri="catalog:default:repl_in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
+ </cluster>
+ </clusters>
+
+ <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
+
+ <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
+ <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-standalone-cluster.xml b/examples/entity/hcat/hcat-standalone-cluster.xml
new file mode 100644
index 0000000..7962b1f
--- /dev/null
+++ b/examples/entity/hcat/hcat-standalone-cluster.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ Example cluster definition that can be used with falcon and standalone
+ hadoop and oozie instances
+-->
+<cluster colo="local" description="Standalone cluster with hacatalog" name="hcat-local" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
+
+ <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+ <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
+
+ <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/falcon/hcat-staging"/>
+ <location name="temp" path="/projects/falcon/hcat-tmp"/>
+ <location name="working" path="/projects/falcon/hcat-working"/>
+ </locations>
+ <properties>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hcat-standalone-target-cluster.xml b/examples/entity/hcat/hcat-standalone-target-cluster.xml
new file mode 100644
index 0000000..8de78cf
--- /dev/null
+++ b/examples/entity/hcat/hcat-standalone-target-cluster.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ Example cluster definition that can be used with falcon and standalone
+ hadoop and oozie instances
+-->
+<cluster colo="hcat-local-target" description="Standalone cluster with hacatalog" name="hcat-local-target" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
+
+ <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+ <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+ <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
+
+ <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
+ </interfaces>
+ <locations>
+ <location name="staging" path="/projects/falcon/hcat-staging-target"/>
+ <location name="temp" path="/projects/falcon/hcat-tmp-target"/>
+ <location name="working" path="/projects/falcon/hcat-working-target"/>
+ </locations>
+ <properties>
+ </properties>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hive-process.xml
----------------------------------------------------------------------
diff --git a/examples/entity/hcat/hive-process.xml b/examples/entity/hcat/hive-process.xml
new file mode 100644
index 0000000..ad8b219
--- /dev/null
+++ b/examples/entity/hcat/hive-process.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="hive-process" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <cluster name="hcat-local">
+ <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
+ </cluster>
+ </clusters>
+
+ <parallel>1</parallel>
+ <order>FIFO</order>
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+
+ <inputs>
+ <!-- In the pig script, the input paths will be available in a variable 'inparts' -->
+ <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/>
+ </inputs>
+
+ <outputs>
+ <!-- In the pig script, the output path will be available in a variable 'outpart' -->
+ <output name="outpart" feed="hcat-out" instance="now(0,0)"/>
+ </outputs>
+
+ <workflow engine="hive" path="/app/hive/wordcount.hql"/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
new file mode 100644
index 0000000..f8f606c
--- /dev/null
+++ b/examples/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-main</artifactId>
+ <version>0.10-SNAPSHOT</version>
+ </parent>
+ <artifactId>falcon-examples</artifactId>
+ <description>Apache Falcon Examples</description>
+ <name>Apache Falcon Examples</name>
+
+ <profiles>
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8f4561c..a72889e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -469,6 +469,7 @@
<module>prism</module>
<module>unit</module>
<module>lifecycle</module>
+ <module>examples</module>
<module>webapp</module>
<module>docs</module>
<module>distro</module>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/assembly-standalone.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/assembly-standalone.xml b/src/main/assemblies/assembly-standalone.xml
index d3111b7..b9d705e 100644
--- a/src/main/assemblies/assembly-standalone.xml
+++ b/src/main/assemblies/assembly-standalone.xml
@@ -99,7 +99,7 @@
</fileSet>
<fileSet>
- <directory>src/main/examples</directory>
+ <directory>examples</directory>
<outputDirectory>examples</outputDirectory>
</fileSet>
@@ -292,4 +292,4 @@
<fileMode>0755</fileMode>
</file>
</files>
-</assembly>
\ No newline at end of file
+</assembly>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/distributed-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
index eb45c6f..a5171c4 100644
--- a/src/main/assemblies/distributed-package.xml
+++ b/src/main/assemblies/distributed-package.xml
@@ -218,10 +218,23 @@
</fileSet>
<fileSet>
- <directory>../src/main/examples</directory>
- <outputDirectory>examples</outputDirectory>
+ <directory>../examples/app</directory>
+ <outputDirectory>examples/app</outputDirectory>
<directoryMode>0755</directoryMode>
</fileSet>
+
+ <fileSet>
+ <directory>../examples/data</directory>
+ <outputDirectory>examples/data</outputDirectory>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>../examples/entity</directory>
+ <outputDirectory>examples/entity</outputDirectory>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index 0b5c69a..b5db01c 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -115,8 +115,20 @@
</fileSet>
<fileSet>
- <directory>../src/main/examples</directory>
- <outputDirectory>examples</outputDirectory>
+ <directory>../examples/app</directory>
+ <outputDirectory>examples/app</outputDirectory>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>../examples/data</directory>
+ <outputDirectory>examples/data</outputDirectory>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>../examples/entity</directory>
+ <outputDirectory>examples/entity</outputDirectory>
<directoryMode>0755</directoryMode>
</fileSet>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/hive/wordcount.hql
----------------------------------------------------------------------
diff --git a/src/main/examples/app/hive/wordcount.hql b/src/main/examples/app/hive/wordcount.hql
deleted file mode 100644
index 4c101c5..0000000
--- a/src/main/examples/app/hive/wordcount.hql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-FROM ${falcon_inparts_table} INSERT OVERWRITE TABLE ${falcon_outpart_table} PARTITION(${falcon_outpart_partitions}) SELECT word, SUM(cnt) as cnt WHERE ${falcon_inparts_filter} GROUP BY word;
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/oozie-mr/workflow.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/app/oozie-mr/workflow.xml b/src/main/examples/app/oozie-mr/workflow.xml
deleted file mode 100644
index c715c91..0000000
--- a/src/main/examples/app/oozie-mr/workflow.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf">
- <start to="mr-node"/>
- <action name="mr-node">
- <map-reduce>
- <job-tracker>${jobTracker}</job-tracker>
- <name-node>${nameNode}</name-node>
- <prepare>
- <delete path="${outpath}"/>
- </prepare>
- <configuration>
- <property>
- <name>mapred.job.queue.name</name>
- <value>${queueName}</value>
- </property>
- <property>
- <name>mapred.mapper.class</name>
- <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
- </property>
- <property>
- <name>mapred.reducer.class</name>
- <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
- </property>
- <property>
- <name>mapred.map.tasks</name>
- <value>1</value>
- </property>
- <property>
- <name>mapred.input.dir</name>
- <value>${inpaths}</value>
- </property>
- <property>
- <name>mapred.output.dir</name>
- <value>${outpath}</value>
- </property>
- </configuration>
- </map-reduce>
- <ok to="end"/>
- <error to="fail"/>
- </action>
- <kill name="fail">
- <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
- </kill>
- <end name="end"/>
-</workflow-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/pig/hcat-wordcount.pig
----------------------------------------------------------------------
diff --git a/src/main/examples/app/pig/hcat-wordcount.pig b/src/main/examples/app/pig/hcat-wordcount.pig
deleted file mode 100644
index 3df93c0..0000000
--- a/src/main/examples/app/pig/hcat-wordcount.pig
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-indata = LOAD '$falcon_inparts_table' USING org.apache.hive.hcatalog.pig.HCatLoader();
-filterdata = FILTER indata BY $falcon_inparts_filter;
-grpdata = GROUP filterdata BY (word);
-finaldata = FOREACH grpdata GENERATE FLATTEN(group) as word, (int)SUM(filterdata.cnt) as cnt;
-STORE finaldata INTO '$falcon_outpart_table' USING org.apache.hive.hcatalog.pig.HCatStorer('$falcon_outpart_partitions');
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/pig/wordcount.pig
----------------------------------------------------------------------
diff --git a/src/main/examples/app/pig/wordcount.pig b/src/main/examples/app/pig/wordcount.pig
deleted file mode 100644
index 190d2d4..0000000
--- a/src/main/examples/app/pig/wordcount.pig
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-indata = LOAD '$inpaths' USING PigStorage() AS (words:chararray);
-tokdata = FOREACH indata GENERATE FLATTEN(TOKENIZE(words)) as word;
-grpdata = GROUP tokdata BY (word);
-finaldata = FOREACH grpdata GENERATE FLATTEN(group), COUNT(tokdata);
-STORE finaldata INTO '$outpath' USING PigStorage('\t');
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/data/generate.sh
----------------------------------------------------------------------
diff --git a/src/main/examples/data/generate.sh b/src/main/examples/data/generate.sh
deleted file mode 100755
index 54db3d7..0000000
--- a/src/main/examples/data/generate.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
- fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR};pwd`
-
-rm -rf generated-data
-YEAR=`date +%Y`
-MONTH=`date +m`
-DAY=`date +%d`
-HOUR=`date +%H`
-
-DELIM='\t'
-input=(first second third fourth fifth)
-for MINUTE in `seq -w 00 59`
-do
- mkdir -p generated-data/00/$MINUTE/
- word=${input[$RANDOM % 5]}
- cnt=`expr $RANDOM % 10`
- echo -e "$word$DELIM$cnt" > generated-data/00/$MINUTE/data
-done
-
-hadoop fs -rmr /data/in/2013/11/15/
-hadoop fs -mkdir -p /data/in/2013/11/15/
-hadoop fs -put generated-data/00 /data/in/2013/11/15/
-rm -rf generated-data
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/data/hcat-generate.sh
----------------------------------------------------------------------
diff --git a/src/main/examples/data/hcat-generate.sh b/src/main/examples/data/hcat-generate.sh
deleted file mode 100644
index 5c4b40a..0000000
--- a/src/main/examples/data/hcat-generate.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
- fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR};pwd`
-
-${BASEDIR}/generate.sh
-
-hcat -e "DROP TABLE IF EXISTS in_table"
-hcat -e "DROP TABLE IF EXISTS repl_in_table"
-hcat -e "DROP TABLE IF EXISTS out_table"
-hcat -e "CREATE TABLE in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
-hcat -e "CREATE TABLE repl_in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
-hcat -e "CREATE TABLE out_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);"
-for MINUTE in `seq -w 00 59`
-do
- hcat -e "ALTER TABLE in_table ADD PARTITION (ds='2013-11-15-00-$MINUTE') LOCATION '/data/in/2013/11/15/00/$MINUTE';"
-done
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/embedded-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/embedded-cluster.xml b/src/main/examples/entity/filesystem/embedded-cluster.xml
deleted file mode 100644
index c505066..0000000
--- a/src/main/examples/entity/filesystem/embedded-cluster.xml
+++ /dev/null
@@ -1,51 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!--
- Example cluster definition that can be used with the embedded
- version of the falcon system started through mvn jetty:run
-
- How to bring up the test falcon instance:
-
- export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
- rm -rf target/store/*
- rm -rf webapp/target/webapps/oozie/data/*
- cd webapp
- mvn jetty:run
--->
-<cluster colo="local" description="" name="local" xmlns="uri:falcon:cluster:0.1">
- <interfaces>
- <interface type="readonly" endpoint="hftp://localhost:41110" version="1.1.2"/>
-
- <interface type="write" endpoint="hdfs://localhost:41020" version="1.1.2"/>
-
- <interface type="execute" endpoint="localhost:41021" version="1.1.2"/>
-
- <interface type="workflow" endpoint="http://localhost:41000/oozie/" version="4.0.0"/>
-
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/staging"/> <!--mandatory-->
- <location name="temp" path="/projects/falcon/tmp"/> <!--optional-->
- <location name="working" path="/projects/falcon/working"/> <!--optional-->
- </locations>
- <properties>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/in-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/in-feed.xml b/src/main/examples/entity/filesystem/in-feed.xml
deleted file mode 100644
index 798de9c..0000000
--- a/src/main/examples/entity/filesystem/in-feed.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="input" name="in" xmlns="uri:falcon:feed:0.1">
- <groups>input</groups>
-
- <frequency>minutes(1)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="local">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- </clusters>
-
- <locations>
- <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
- </locations>
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/log/log.format.csv" provider="csv"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/oozie-mr-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/oozie-mr-process.xml b/src/main/examples/entity/filesystem/oozie-mr-process.xml
deleted file mode 100644
index c080436..0000000
--- a/src/main/examples/entity/filesystem/oozie-mr-process.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<process name="oozie-mr-process" xmlns="uri:falcon:process:0.1">
- <clusters>
- <cluster name="local">
- <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
- </cluster>
- </clusters>
-
- <parallel>1</parallel>
- <order>FIFO</order>
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
-
- <inputs>
- <!-- In the workflow, the input paths will be available in a variable 'inpaths' -->
- <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/>
- </inputs>
-
- <outputs>
- <!-- In the workflow, the output path will be available in a variable 'outpath' -->
- <output name="outpath" feed="out" instance="now(0,0)"/>
- </outputs>
-
- <properties>
- <!-- In the workflow, these properties will be available with variable - key -->
- <property name="queueName" value="default"/>
- <!-- The schedule time available as a property in workflow -->
- <property name="time" value="${instanceTime()}"/>
- </properties>
-
- <workflow engine="oozie" path="/app/oozie-mr"/>
-</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/out-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/out-feed.xml b/src/main/examples/entity/filesystem/out-feed.xml
deleted file mode 100644
index f537bbc..0000000
--- a/src/main/examples/entity/filesystem/out-feed.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="output" name="out" xmlns="uri:falcon:feed:0.1">
- <groups>output</groups>
-
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="local">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- </clusters>
-
- <locations>
- <location type="data" path="/data/out/${YEAR}/${MONTH}/${DAY}/${HOUR}"/>
- </locations>
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/out/out.format.csv" provider="csv"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/pig-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/pig-process.xml b/src/main/examples/entity/filesystem/pig-process.xml
deleted file mode 100644
index 99e03bd..0000000
--- a/src/main/examples/entity/filesystem/pig-process.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<process name="pig-process" xmlns="uri:falcon:process:0.1">
- <clusters>
- <cluster name="local">
- <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
- </cluster>
- </clusters>
-
- <parallel>1</parallel>
- <order>FIFO</order>
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
-
- <inputs>
- <!-- In the pig script, the input paths will be available in a variable 'inpaths' -->
- <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/>
- </inputs>
-
- <outputs>
- <!-- In the pig script, the output path will be available in a variable 'outpath' -->
- <output name="outpath" feed="out" instance="now(0,0)"/>
- </outputs>
-
- <workflow engine="pig" path="/app/pig/wordcount.pig"/>
-</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/replication-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/replication-feed.xml b/src/main/examples/entity/filesystem/replication-feed.xml
deleted file mode 100644
index 7387d45..0000000
--- a/src/main/examples/entity/filesystem/replication-feed.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="replication feed" name="repl-feed" xmlns="uri:falcon:feed:0.1">
- <groups>input</groups>
-
- <frequency>minutes(1)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="local" type="source">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- <cluster name="local-target" type="target">
- <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- <locations>
- <location type="data" path="/data/repl-in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
- </locations>
- </cluster>
- </clusters>
-
- <locations>
- <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
- </locations>
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/log/log.format.csv" provider="csv"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/standalone-cluster.xml b/src/main/examples/entity/filesystem/standalone-cluster.xml
deleted file mode 100644
index 4f8a5fc..0000000
--- a/src/main/examples/entity/filesystem/standalone-cluster.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!--
- Example cluster definition that can be used with falcon and standalone
- hadoop and oozie instances
--->
-<cluster colo="local" description="Standalone cluster" name="local" xmlns="uri:falcon:cluster:0.1">
- <interfaces>
- <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
-
- <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
-
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/staging"/> <!--mandatory-->
- <location name="temp" path="/projects/falcon/tmp"/> <!--optional-->
- <location name="working" path="/projects/falcon/working"/> <!--optional-->
- </locations>
- <properties>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/standalone-target-cluster.xml b/src/main/examples/entity/filesystem/standalone-target-cluster.xml
deleted file mode 100644
index cb5c4c6..0000000
--- a/src/main/examples/entity/filesystem/standalone-target-cluster.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<!--
- Example cluster definition that can be used with falcon and standalone
- hadoop and oozie instances
--->
-<cluster colo="local" description="Standalone cluster" name="local-target" xmlns="uri:falcon:cluster:0.1">
- <interfaces>
- <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
-
- <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
-
- <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
-
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/staging-target"/> <!--mandatory-->
- <location name="temp" path="/projects/falcon/tmp-target"/> <!--optional-->
- <location name="working" path="/projects/falcon/working-target"/> <!--optional-->
- </locations>
- <properties>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-in-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-in-feed.xml b/src/main/examples/entity/hcat/hcat-in-feed.xml
deleted file mode 100644
index f978e8c..0000000
--- a/src/main/examples/entity/hcat/hcat-in-feed.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="input" name="hcat-in" xmlns="uri:falcon:feed:0.1">
- <groups>input</groups>
-
- <frequency>minutes(1)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="hcat-local">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- </clusters>
-
- <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/log/log.format.csv" provider="csv"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-out-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-out-feed.xml b/src/main/examples/entity/hcat/hcat-out-feed.xml
deleted file mode 100644
index 23b22b9..0000000
--- a/src/main/examples/entity/hcat/hcat-out-feed.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="output" name="hcat-out" xmlns="uri:falcon:feed:0.1">
- <frequency>minutes(5)</frequency>
- <timezone>UTC</timezone>
- <late-arrival cut-off="hours(1)"/>
-
- <clusters>
- <cluster name="hcat-local">
- <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
- <retention limit="hours(2)" action="delete"/>
- </cluster>
- </clusters>
-
- <table uri="catalog:default:out_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" />
-
- <ACL owner="testuser-ut-user" group="group" permission="0x644"/>
- <schema location="/schema/out/out.format.csv" provider="csv"/>
-</feed>