You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/07/19 18:43:54 UTC
[1/2] falcon git commit: FALCON-2066 Extend support for Import and
Export of various database types
Repository: falcon
Updated Branches:
refs/heads/master 9a4c18c7d -> 292dfed34
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
index fafd56e..3e8ca73 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
@@ -27,6 +27,7 @@ import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.datasource.Credential;
import org.apache.falcon.entity.v0.datasource.Credentialtype;
import org.apache.falcon.entity.v0.datasource.Datasource;
+import org.apache.falcon.entity.v0.datasource.DatasourceType;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.oozie.sqoop.ACTION;
import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
@@ -60,9 +61,8 @@ public final class ImportExportCommon {
private ImportExportCommon() {
}
- static StringBuilder buildUserPasswordArg(StringBuilder builder, StringBuilder sqoopOpts, Datasource db)
+ static StringBuilder buildUserPasswordArg(StringBuilder builder, StringBuilder sqoopOpts, Credential cred)
throws FalconException {
- Credential cred = DatasourceHelper.getReadPasswordInfo(db);
builder.append("--username").append(ARG_SEPARATOR)
.append(cred.getUserName())
.append(ARG_SEPARATOR);
@@ -111,4 +111,22 @@ public final class ImportExportCommon {
return Splitter.on(";").withKeyValueSeparator("=").split(partitionStr);
}
+ public static StringBuilder buildDriverArgs(StringBuilder builder, Datasource db)
+ throws FalconException {
+ if ((db.getType() == DatasourceType.GENERIC) && (db.getDriver() != null)
+ && (db.getDriver().getClazz() != null)) {
+ builder.append("--driver").append(ImportExportCommon.ARG_SEPARATOR).append(db.getDriver().getClazz());
+ }
+ return builder;
+ }
+
+ public static StringBuilder buildConnectArg(StringBuilder builder, String endPoint) throws FalconException {
+ return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR)
+ .append(endPoint);
+ }
+
+ public static StringBuilder buildTableArg(StringBuilder builder, String tableName) throws FalconException {
+ return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR)
+ .append(tableName);
+ }
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
index 0801899..5ad3d03 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
@@ -29,7 +29,6 @@ import org.apache.falcon.entity.HiveUtil;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.ClusterLocationType;
-import org.apache.falcon.entity.v0.datasource.DatasourceType;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.hadoop.HadoopClientFactory;
@@ -150,26 +149,10 @@ public abstract class OozieOrchestrationWorkflowBuilder<T extends Entity> extend
}
case IMPORT:
- DatasourceType dsType = EntityUtil.getImportDatasourceType(cluster, feed);
- if ((dsType == DatasourceType.MYSQL)
- || (dsType == DatasourceType.ORACLE)
- || (dsType == DatasourceType.HSQL)) {
- return new DatabaseImportWorkflowBuilder(feed);
- } else {
- LOG.info("Import policy not implemented for DataSourceType : " + dsType);
- }
- break;
+ return new DatabaseImportWorkflowBuilder(feed);
case EXPORT:
- dsType = EntityUtil.getExportDatasourceType(cluster, feed);
- if ((dsType == DatasourceType.MYSQL)
- || (dsType == DatasourceType.ORACLE)
- || (dsType == DatasourceType.HSQL)) {
- return new DatabaseExportWorkflowBuilder(feed);
- } else {
- LOG.info("Export policy not implemented for DataSourceType : " + dsType);
- }
- break;
+ return new DatabaseExportWorkflowBuilder(feed);
default:
throw new IllegalArgumentException("Unhandled type " + entity.getEntityType()
[2/2] falcon git commit: FALCON-2066 Extend support for Import and
Export of various database types
Posted by ba...@apache.org.
FALCON-2066 Extend support for Import and Export of various database types
This is a replacement for pull request - https://github.com/apache/falcon/pull/224.
+ Checkstyle violations addressed
+ Removed the changes that went into pull-218
+ Created https://issues.apache.org/jira/browse/FALCON-2083 to track the direct mode & verbose
+ Removed tab from the xml files
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Reviewers: "Peeyush B <pb...@hortonworks.com>, Balu Vellanki <ba...@apache.org>"
Closes #229 from vramachan/FALCON-2066.DatabaseTypes
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/292dfed3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/292dfed3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/292dfed3
Branch: refs/heads/master
Commit: 292dfed341b462488ab58584dc305f519f1beb4d
Parents: 9a4c18c
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Authored: Tue Jul 19 11:43:50 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Tue Jul 19 11:43:50 2016 -0700
----------------------------------------------------------------------
client/src/main/resources/datasource-0.1.xsd | 5 ++
docs/src/site/twiki/EntitySpecification.twiki | 8 ++
examples/entity/importexport/README.txt | 56 ++++++++++++
examples/entity/importexport/db2-datasource.xml | 56 ++++++++++++
.../importexport/db2-feed-import-filesystem.xml | 71 ++++++++++++++++
.../importexport/feed-export-filesystem.xml | 63 ++++++++++++++
.../entity/importexport/feed-export-hcat.xml | 65 ++++++++++++++
.../feed-export-repl-filesystem.xml | 72 ++++++++++++++++
.../feed-import-export-filesystem.xml | 80 ++++++++++++++++++
.../feed-import-export-repl-filesystem.xml | 89 ++++++++++++++++++++
.../entity/importexport/feed-import-hcat.xml | 69 +++++++++++++++
.../feed-import-repl-filesystem.xml | 81 ++++++++++++++++++
.../entity/importexport/generic-datasource.xml | 49 +++++++++++
.../generic-feed-import-filesystem.xml | 72 ++++++++++++++++
.../entity/importexport/mysql-datasource.xml | 58 +++++++++++++
.../mysql-feed-export-filesystem.xml | 62 ++++++++++++++
.../mysql-feed-import-filesystem.xml | 72 ++++++++++++++++
.../entity/importexport/netezza-datasource.xml | 57 +++++++++++++
.../netezza-feed-import-filesystem.xml | 72 ++++++++++++++++
.../entity/importexport/oracle-datasource.xml | 50 +++++++++++
.../oracle-feed-import-filesystem.xml | 72 ++++++++++++++++
.../entity/importexport/pgsql-datasource.xml | 55 ++++++++++++
.../pgsql-feed-import-filesystem.xml | 70 +++++++++++++++
.../entity/importexport/primary_cluster.xml | 33 ++++++++
examples/entity/importexport/td-datasource.xml | 56 ++++++++++++
.../importexport/td-feed-import-filesystem.xml | 70 +++++++++++++++
.../oozie/DatabaseExportWorkflowBuilder.java | 28 ++----
.../oozie/DatabaseImportWorkflowBuilder.java | 37 ++------
.../apache/falcon/oozie/ImportExportCommon.java | 22 ++++-
.../OozieOrchestrationWorkflowBuilder.java | 21 +----
30 files changed, 1603 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/client/src/main/resources/datasource-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/datasource-0.1.xsd b/client/src/main/resources/datasource-0.1.xsd
index ef78239..d78d967 100644
--- a/client/src/main/resources/datasource-0.1.xsd
+++ b/client/src/main/resources/datasource-0.1.xsd
@@ -232,6 +232,11 @@
<xs:enumeration value="mysql"/>
<xs:enumeration value="oracle"/>
<xs:enumeration value="hsql"/>
+ <xs:enumeration value="postgres"/>
+ <xs:enumeration value="db2"/>
+ <xs:enumeration value="netezza"/>
+ <xs:enumeration value="teradata"/>
+ <xs:enumeration value="generic"/>
</xs:restriction>
</xs:simpleType>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/docs/src/site/twiki/EntitySpecification.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki
index faad305..2615e5d 100644
--- a/docs/src/site/twiki/EntitySpecification.twiki
+++ b/docs/src/site/twiki/EntitySpecification.twiki
@@ -117,6 +117,14 @@ Following are the tags defined in a datasource.xml:
name="test-hsql-db" xmlns="uri:falcon:datasource:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
</verbatim>
+---+++ Datasource Types
+
+Falcon currently supports relational databases as data sources (both source and target). The following relational
+databases are supported
+ * MySQL, HSQL, Postgres, Oracle, Teradata, Netezza, DB2
+ * Generic - generic jdbc data source. This will require specifying a driver classname and jar file in the
+ datasource entity specification. Please see samples in the examples dir.
+
The colo specifies the colo to which the datasource belongs to and name is the name of the datasource which has to
be unique.
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/README.txt
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/README.txt b/examples/entity/importexport/README.txt
new file mode 100755
index 0000000..8b20aa5
--- /dev/null
+++ b/examples/entity/importexport/README.txt
@@ -0,0 +1,56 @@
+####### Using MySQL as Datasource #######
+yum install mysql mysql-server mysql-libs
+/usr/share/java/mysql-connector-java.jar
+cp /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/
+unlink /usr/share/java/mysql-connector-java.jar
+ln -s /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/mysql-connector-java.jar
+service mysqld start
+
+####### oozie sharelib change for sqoop ######################
+# Copy /usr/share/java/mysql-connector-java.jar (version 5.1.31) into oozie share lib of sqoop dir
+hadoop dfs -put /usr/share/java/mysql-connector-java-5.1.31.jar /user/oozie/share/lib/lib_XXXXXXXXXXX/sqoop
+
+####### Setup MySQL user and create source table ##############
+# Setup - Create MySQL source table - db_raw_data
+###############################################################
+use test;
+create user sqoop_user@localhost identified by 'sqoop';
+grant all privileges on *.* to 'sqoop_user'@'localhost' with grant option;
+create user 'sqoop_user'@'%' identified by 'sqoop';
+grant all privileges on *.* to 'sqoop_user'@'%' with grant option;
+flush privileges;
+
+## Import: following is the data source tables from which data will be pulled into HADOOP
+
+create table db_raw_data(id int not null, name varchar(20), value int, modified_ts timestamp);
+insert into db_raw_data values (1, 'Apple', 700, now() );
+insert into db_raw_data values (2, 'Blackberry', 1, now() );
+insert into db_raw_data values (3, 'Cisco', 100, now() );
+insert into db_raw_data values (4, 'Delta', 10, now() );
+insert into db_raw_data values (5, 'Eagle', 55, now() );
+insert into db_raw_data values (6, 'Falcon', 99, now() );
+
+## submit the following falcon entities and schedule
+
+falcon entity -type cluster -file primary_cluster.xml -submit
+falcon entity -type datasource -file mysql-datasource.xml -submit
+falcon entity -type feed -file mysql-feed-import-filesystem.xml -submitAndSchedule
+
+## Export: create a target table in the MySQL database
+
+create table db_export_fs(id int not null, name varchar(20), value int, modified_ts timestamp);
+
+## submit the following falcon entities and schedule
+falcon entity -type feed -file mysql-feed-export-filesystem.xml -submitAndSchedule
+
+### Note
+# For other supported database types like Oracle, Postgres, Teradata, Netezza, DB2, Generic JDB
+# please use the corresponding feed entity that starts with the database name in the xml entity
+# definition.
+
+# Also, please make sure to copy the jdbc driver onto the Oozie Sqoop shared lib on HDFS and
+# restart Oozie. The datasource entity definition will also refer to the JDBC drivers in order
+# for falcon to validate the connection when submitting the datasource entity.
+
+
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/db2-datasource.xml b/examples/entity/importexport/db2-datasource.xml
new file mode 100755
index 0000000..36a061b
--- /dev/null
+++ b/examples/entity/importexport/db2-datasource.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="DB2 database on west coast" type="db2" name="db2-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:db2://172.25.16.65:50000/Sample">
+ <credential type="password-text">
+ <userName>sqoop</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:db2://172.25.16.65:50000/Sample">
+ <credential type="password-text">
+ <userName>sqoop</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** default credential ***** -->
+ <credential type="password-alias">
+ <userName>sqoop</userName>
+ <passwordAlias>
+ <alias>sqoop.password.alias</alias>
+ <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+ </passwordAlias>
+ </credential>
+
+ </interfaces>
+
+ <driver>
+ <clazz>com.ibm.db2.jcc.DB2Driver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/db2jcc4.jar</jar>
+ </driver>
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/db2-feed-import-filesystem.xml b/examples/entity/importexport/db2-feed-import-filesystem.xml
new file mode 100755
index 0000000..2adca37
--- /dev/null
+++ b/examples/entity/importexport/db2-feed-import-filesystem.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="db2-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="db2-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/db2-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-filesystem.xml b/examples/entity/importexport/feed-export-filesystem.xml
new file mode 100755
index 0000000..36d2c79
--- /dev/null
+++ b/examples/entity/importexport/feed-export-filesystem.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <export>
+ <target name="mysql-db" tableName="simple_export_filesystem">
+ <load type="allowinsert"/>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-hcat.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-hcat.xml b/examples/entity/importexport/feed-export-hcat.xml
new file mode 100755
index 0000000..f52bf20
--- /dev/null
+++ b/examples/entity/importexport/feed-export-hcat.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-hcat" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <export>
+ <target name="mysql-db" tableName="db_processed_data">
+ <load type="allowinsert"/>
+ <!--fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields-->
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ </clusters>
+
+ <table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-repl-filesystem.xml b/examples/entity/importexport/feed-export-repl-filesystem.xml
new file mode 100755
index 0000000..1932b5b
--- /dev/null
+++ b/examples/entity/importexport/feed-export-repl-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <export>
+ <target name="mysql-db" tableName="simple_export_filesystem">
+ <load type="allowinsert"/>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ <cluster name="backupCluster" type="target">
+ <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-export-filesystem.xml b/examples/entity/importexport/feed-import-export-filesystem.xml
new file mode 100755
index 0000000..4e9e215
--- /dev/null
+++ b/examples/entity/importexport/feed-import-export-filesystem.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-export-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="mysql-db" tableName="simple">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ <export>
+ <target name="mysql-db" tableName="simple_export">
+ <load type="allowinsert"/>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-export-repl-filesystem.xml b/examples/entity/importexport/feed-import-export-repl-filesystem.xml
new file mode 100755
index 0000000..4e9a9be
--- /dev/null
+++ b/examples/entity/importexport/feed-import-export-repl-filesystem.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-export-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="mysql-db" tableName="simple">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ <export>
+ <target name="mysql-db" tableName="simple_import_export_filesystem">
+ <load type="allowinsert"/>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ <cluster name="backupCluster" type="target">
+ <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-hcat.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-hcat.xml b/examples/entity/importexport/feed-import-hcat.xml
new file mode 100755
index 0000000..73d2bf4
--- /dev/null
+++ b/examples/entity/importexport/feed-import-hcat.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-hcat" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="mysql-db" tableName="mysql_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <!--<table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/>-->
+ <table uri="catalog:default:hcat_raw_data#year=BLAH"/>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-repl-filesystem.xml b/examples/entity/importexport/feed-import-repl-filesystem.xml
new file mode 100755
index 0000000..2bc110a
--- /dev/null
+++ b/examples/entity/importexport/feed-import-repl-filesystem.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="mysql-db" tableName="mysql_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ <cluster name="backupCluster" type="target">
+ <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/generic-datasource.xml b/examples/entity/importexport/generic-datasource.xml
new file mode 100755
index 0000000..74f15ca
--- /dev/null
+++ b/examples/entity/importexport/generic-datasource.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="Generic database on west coast" type="generic" name="generic-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:mysql://c6402/test">
+ <credential type="password-text">
+ <userName>sqoop_user</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:mysql://c6402/test">
+ <credential type="password-text">
+ <userName>sqoop2_user</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ </interfaces>
+
+ <driver>
+ <clazz>com.mysql.jdbc.Driver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar>
+ </driver>
+
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/generic-feed-import-filesystem.xml b/examples/entity/importexport/generic-feed-import-filesystem.xml
new file mode 100755
index 0000000..01ccdab
--- /dev/null
+++ b/examples/entity/importexport/generic-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="generic-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="generic-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/generic-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-datasource.xml b/examples/entity/importexport/mysql-datasource.xml
new file mode 100755
index 0000000..293b0bb
--- /dev/null
+++ b/examples/entity/importexport/mysql-datasource.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="MySQL database on west coast" type="mysql" name="mysql-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:mysql://c6402/test">
+ <credential type="password-text">
+ <userName>sqoop_user</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:mysql://c6402/test">
+ <credential type="password-text">
+ <userName>sqoop2_user</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** default credential ***** -->
+ <credential type="password-alias">
+ <userName>sqoop2_user</userName>
+ <passwordAlias>
+ <alias>sqoop.password.alias</alias>
+ <providerPath>hdfs://c6407.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+ </passwordAlias>
+ </credential>
+
+ </interfaces>
+
+ <driver>
+ <clazz>com.mysql.jdbc.Driver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar>
+ </driver>
+
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-feed-export-filesystem.xml b/examples/entity/importexport/mysql-feed-export-filesystem.xml
new file mode 100755
index 0000000..2e07aa2
--- /dev/null
+++ b/examples/entity/importexport/mysql-feed-export-filesystem.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->#
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="mysql-feed-export-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <export>
+ <target name="mysql-db" tableName="db_export_fs">
+ <load type="allowinsert"/>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-feed-import-filesystem.xml b/examples/entity/importexport/mysql-feed-import-filesystem.xml
new file mode 100755
index 0000000..b02b86f
--- /dev/null
+++ b/examples/entity/importexport/mysql-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="mysql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="mysql-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/netezza-datasource.xml b/examples/entity/importexport/netezza-datasource.xml
new file mode 100755
index 0000000..1e4dfe6
--- /dev/null
+++ b/examples/entity/importexport/netezza-datasource.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="Netezza database on west coast" type="netezza" name="netezza-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:netezza://10.10.20.14:5480/test">
+ <credential type="password-text">
+ <userName>admin</userName>
+ <passwordText>password</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:netezza://10.10.20.14:5480/test">
+ <credential type="password-text">
+ <userName>admin</userName>
+ <passwordText>password</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** default credential ***** -->
+ <credential type="password-alias">
+ <userName>admin</userName>
+ <passwordAlias>
+ <alias>sqoop.password.alias</alias>
+ <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+ </passwordAlias>
+ </credential>
+
+ </interfaces>
+
+ <driver>
+ <clazz>org.netezza.Driver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/nzjdbc.jar</jar>
+ </driver>
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/netezza-feed-import-filesystem.xml b/examples/entity/importexport/netezza-feed-import-filesystem.xml
new file mode 100755
index 0000000..bbbcbad
--- /dev/null
+++ b/examples/entity/importexport/netezza-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="netezza-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="netezza-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/netezza-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/oracle-datasource.xml b/examples/entity/importexport/oracle-datasource.xml
new file mode 100755
index 0000000..535640c
--- /dev/null
+++ b/examples/entity/importexport/oracle-datasource.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="Oracle database on west coast" type="oracle" name="oracle-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE">
+ <credential type="password-text">
+ <userName>SQOOP</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE">
+ <credential type="password-text">
+ <userName>SQOOP</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ </interfaces>
+
+ <driver>
+ <clazz>oracle.jdbc.OracleDriver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/ojdbc6.jar</jar>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/orai18n.jar</jar>
+ </driver>
+
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/oracle-feed-import-filesystem.xml b/examples/entity/importexport/oracle-feed-import-filesystem.xml
new file mode 100755
index 0000000..5bba3de
--- /dev/null
+++ b/examples/entity/importexport/oracle-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="oracle-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="oracle-db" tableName="DB_RAW_DATA">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>ID</field>
+ <field>NAME</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="ID"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/oracle-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/pgsql-datasource.xml b/examples/entity/importexport/pgsql-datasource.xml
new file mode 100755
index 0000000..b161bd3
--- /dev/null
+++ b/examples/entity/importexport/pgsql-datasource.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<datasource colo="west-coast" description="Postgres database on west coast" type="postgres" name="pgsql-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:postgresql://c6402:5432/test">
+ <credential type="password-text">
+ <userName>ambari-qa</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:postgresql://c6402:5432/test">
+ <credential type="password-text">
+ <userName>ambari-qa</userName>
+ <passwordText>sqoop</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** default credential ***** -->
+ <credential type="password-alias">
+ <userName>ambari-qa</userName>
+ <passwordAlias>
+ <alias>sqoop.password.alias</alias>
+ <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+ </passwordAlias>
+ </credential>
+ </interfaces>
+
+ <driver>
+ <clazz>org.postgresql.Driver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/postgresql-9.4.1208.jar</jar>
+ </driver>
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/pgsql-feed-import-filesystem.xml b/examples/entity/importexport/pgsql-feed-import-filesystem.xml
new file mode 100755
index 0000000..fa88cd1
--- /dev/null
+++ b/examples/entity/importexport/pgsql-feed-import-filesystem.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="pgsql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="pgsql-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/pgsql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/primary_cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/primary_cluster.xml b/examples/entity/importexport/primary_cluster.xml
new file mode 100755
index 0000000..4d43fd7
--- /dev/null
+++ b/examples/entity/importexport/primary_cluster.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<cluster colo="USWestOregon" description="oregonHadoopCluster" name="primaryCluster" xmlns="uri:falcon:cluster:0.1">
+ <interfaces>
+ <interface type="readonly" endpoint="webhdfs://localhost:50070" version="2.2.0" />
+ <interface type="write" endpoint="hdfs://localhost:8020" version="2.2.0" />
+ <interface type="execute" endpoint="localhost:8050" version="2.2.0" />
+ <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0" />
+ <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.1.6" />
+ <interface type="registry" endpoint="thrift://localhost:9083" version="0.14.0" />
+ </interfaces>
+ <locations>
+ <location name="staging" path="/apps/falcon/primaryCluster/staging" />
+ <location name="temp" path="/tmp" />
+ <location name="working" path="/apps/falcon/primaryCluster/working" />
+ </locations>
+</cluster>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/td-datasource.xml b/examples/entity/importexport/td-datasource.xml
new file mode 100755
index 0000000..a9eb4ed
--- /dev/null
+++ b/examples/entity/importexport/td-datasource.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<datasource colo="west-coast" description="Teradata database on west coast" type="teradata" name="td-db" xmlns="uri:falcon:datasource:0.1">
+ <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+ <interfaces>
+
+ <!-- ***** read interface ***** -->
+ <interface type="readonly" endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail">
+ <credential type="password-text">
+ <userName>dbc</userName>
+ <passwordText>dbc</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** write interface ***** -->
+ <interface type="write" endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail">
+ <credential type="password-text">
+ <userName>dbc</userName>
+ <passwordText>dbc</passwordText>
+ </credential>
+ </interface>
+
+ <!-- ***** default credential ***** -->
+ <credential type="password-alias">
+ <userName>dbc</userName>
+ <passwordAlias>
+ <alias>sqoop.password.alias</alias>
+ <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+ </passwordAlias>
+ </credential>
+
+ </interfaces>
+
+ <driver>
+ <clazz>com.teradata.jdbc.TeraDriver</clazz>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/terajdbc4.jar</jar>
+ <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/tdgssconfig.jar</jar>
+ </driver>
+</datasource>
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/td-feed-import-filesystem.xml b/examples/entity/importexport/td-feed-import-filesystem.xml
new file mode 100755
index 0000000..6dd94ed
--- /dev/null
+++ b/examples/entity/importexport/td-feed-import-filesystem.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="td-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+ <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+ <groups>DataImportPipeline</groups>
+ <frequency>minutes(10)</frequency>
+ <late-arrival cut-off="hours(4)"/>
+ <clusters>
+ <cluster name="primaryCluster" type="source">
+ <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+ <retention limit="days(90)" action="delete"/>
+ <import>
+ <source name="td-db" tableName="db_raw_data">
+ <extract type="full">
+ <mergepolicy>snapshot</mergepolicy>
+ </extract>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </source>
+ <arguments>
+ <argument name="--split-by" value="id"/>
+ <argument name="--num-mappers" value="2"/>
+ </arguments>
+ </import>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/user/ambari-qa/falcon/demo/primary/td-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+ <location type="stats" path="/none"/>
+ <location type="meta" path="/none"/>
+ </locations>
+
+ <ACL owner="ambari-qa" group="users" permission="0755"/>
+ <schema location="/none" provider="none"/>
+
+ <lifecycle>
+ <retention-stage>
+ <frequency>days(1)</frequency>
+ <queue>default</queue>
+ <priority>LOW</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
index e7ca869..93f3d1f 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
@@ -94,12 +94,15 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder {
StringBuilder sqoopArgs = new StringBuilder();
StringBuilder sqoopOptions = new StringBuilder();
- buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
- buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
- Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(
- FeedHelper.getCluster(entity, cluster.getName())));
- ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource)
- .append(ImportExportCommon.ARG_SEPARATOR);
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
+ Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster));
+ ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getWriteEndpoint(datasource))
+ .append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getExportDataSourceTableName(feedCluster))
+ .append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions,
+ DatasourceHelper.getWritePasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR);
buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
buildArguments(sqoopArgs, extraArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR);
buildLoadType(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
@@ -111,19 +114,6 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder {
.append(sqoopArgs).toString();
}
- private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException {
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
- return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR)
- .append(DatasourceHelper.getReadOnlyEndpoint(
- DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster))));
- }
-
- private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException {
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
- return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR)
- .append(FeedHelper.getExportDataSourceTableName(feedCluster));
- }
-
private StringBuilder buildLoadType(StringBuilder builder, Cluster cluster)
throws FalconException {
org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
index 1efa368..77d5462 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
@@ -95,13 +95,16 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder {
Map<String, String> extraArgs = getArguments(cluster);
StringBuilder sqoopArgs = new StringBuilder();
StringBuilder sqoopOptions = new StringBuilder();
- buildDriverArgs(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
- buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
- buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
- Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(
- FeedHelper.getCluster(entity, cluster.getName())));
- ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource)
+
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
+ Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster));
+ ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getReadOnlyEndpoint(datasource))
.append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getImportDataSourceTableName(feedCluster))
+ .append(ImportExportCommon.ARG_SEPARATOR);
+ ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions,
+ DatasourceHelper.getReadPasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR);
buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
buildArguments(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
buildTargetArg(sqoopArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR);
@@ -112,28 +115,6 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder {
.append(sqoopArgs).toString();
}
- private StringBuilder buildDriverArgs(StringBuilder builder, Cluster cluster) throws FalconException {
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
- Datasource db = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster));
- if ((db.getDriver() != null) && (db.getDriver().getClazz() != null)) {
- builder.append("--driver").append(ImportExportCommon.ARG_SEPARATOR).append(db.getDriver().getClazz());
- }
- return builder;
- }
-
- private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException {
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
- return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR)
- .append(DatasourceHelper.getReadOnlyEndpoint(
- DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster))));
- }
-
- private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException {
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
- return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR)
- .append(FeedHelper.getImportDataSourceTableName(feedCluster));
- }
-
private StringBuilder buildTargetArg(StringBuilder builder, Feed feed, Cluster cluster)
throws FalconException {
Storage.TYPE feedStorageType = FeedHelper.getStorageType(feed, cluster);