You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/04/06 22:40:07 UTC
falcon git commit: FALCON-1881 Database export does not require
fields tag in entity specification
Repository: falcon
Updated Branches:
refs/heads/master 14b1bb8f4 -> fa66c17e9
FALCON-1881 Database export does not require fields tag in entity specification
Database export does not require fields tag in entity specification
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Reviewers: "Peeyush Bishnoi <pe...@apache.org>, Pallavi Rao <pa...@inmobi.com>, Balu Vellanki <bv...@hortonworks.com>"
Closes #83 from vramachan/FALCON-1881.20160331-1.ExportFieldsList
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/fa66c17e
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/fa66c17e
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/fa66c17e
Branch: refs/heads/master
Commit: fa66c17e967359c519c59717f52a705297595d3f
Parents: 14b1bb8
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Authored: Wed Apr 6 13:39:42 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Wed Apr 6 13:39:42 2016 -0700
----------------------------------------------------------------------
.../falcon/entity/parser/FeedEntityParser.java | 16 +++--
.../entity/parser/FeedEntityParserTest.java | 4 +-
.../feed/feed-export-exclude-fields-0.1.xml | 66 --------------------
.../config/feed/feed-export-fields-0.1.xml | 66 ++++++++++++++++++++
.../test/resources/feed-export-template6.xml | 6 --
5 files changed, 79 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index c942862..2a9a852 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -110,7 +110,7 @@ public class FeedEntityParser extends EntityParser<Feed> {
if (FeedHelper.isExportEnabled(cluster)) {
validateEntityExists(EntityType.DATASOURCE, FeedHelper.getExportDatasourceName(cluster));
validateFeedExportArgs(cluster);
- validateFeedExportFieldExcludes(cluster);
+ validateFeedExportFields(cluster);
}
}
@@ -709,10 +709,16 @@ public class FeedEntityParser extends EntityParser<Feed> {
}
}
- private void validateFeedExportFieldExcludes(Cluster feedCluster) throws FalconException {
- if (FeedHelper.isFieldExcludes(feedCluster.getExport().getTarget())) {
- throw new ValidationException(String.format("Field excludes are not supported "
- + "currently in Feed import policy"));
+ /**
+ * Export infers the target fields from the destination. There is no need to enumerate or exclude the fields
+ * in the feed entity definition.
+ *
+ * @param feedCluster feed's cluster
+ * @throws FalconException
+ */
+ private void validateFeedExportFields(Cluster feedCluster) throws FalconException {
+ if (feedCluster.getExport().getTarget().getFields() != null) {
+ throw new ValidationException(String.format("Feed Export does not expect Fields specification"));
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 382a8c4..ceec3c4 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -1213,9 +1213,9 @@ public class FeedEntityParserTest extends AbstractTestBase {
public void testExportFeedSqoopExcludeFields() throws Exception {
storeEntity(EntityType.CLUSTER, "testCluster");
- InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-exclude-fields-0.1.xml");
+ InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-fields-0.1.xml");
Feed feed = parser.parseAndValidate(feedStream);
- Assert.fail("An exception should have been thrown: Feed Export policy not yet implement Field exclusion.");
+ Assert.fail("An exception should have been thrown: Feed Export policy does not require Fields specification.");
}
@Test (expectedExceptions = ValidationException.class)
http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
deleted file mode 100644
index 6753a00..0000000
--- a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1">
- <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, _department_type=forecasting</tags>
- <partitions>
- <partition name="fraud"/>
- <partition name="good"/>
- </partitions>
-
- <groups>online,bi</groups>
- <availabilityFlag>_SUCCESS</availabilityFlag>
-
- <frequency>hours(1)</frequency>
- <sla slaLow="hours(2)" slaHigh="hours(3)"/>
- <timezone>UTC</timezone>
-
- <late-arrival cut-off="hours(6)"/>
-
- <clusters>
- <cluster name="testCluster" type="source">
- <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/>
- <retention limit="hours(48)" action="delete"/>
- <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
- <sla slaLow="hours(3)" slaHigh="hours(4)"/>
- <export>
- <target name="test-hsql-db" tableName="customer">
- <load type="updateonly"/>
- <fields>
- <excludes>
- <field>id</field>
- <field>name</field>
- </excludes>
- </fields>
- </target>
- <arguments>
- <argument name="--update-key" value="id"/>
- </arguments>
- </export>
- </cluster>
- </clusters>
-
- <locations>
- <location type="data" path="/projects/falcon/clicks"/>
- <location type="stats" path="/projects/falcon/clicksStats"/>
- <location type="meta" path="/projects/falcon/clicksMetaData"/>
- </locations>
-
- <ACL owner="testuser" group="group" permission="0x755"/>
- <schema location="/schema/clicks" provider="protobuf"/>
-</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/feed/feed-export-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
new file mode 100644
index 0000000..8fda43d
--- /dev/null
+++ b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1">
+ <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, _department_type=forecasting</tags>
+ <partitions>
+ <partition name="fraud"/>
+ <partition name="good"/>
+ </partitions>
+
+ <groups>online,bi</groups>
+ <availabilityFlag>_SUCCESS</availabilityFlag>
+
+ <frequency>hours(1)</frequency>
+ <sla slaLow="hours(2)" slaHigh="hours(3)"/>
+ <timezone>UTC</timezone>
+
+ <late-arrival cut-off="hours(6)"/>
+
+ <clusters>
+ <cluster name="testCluster" type="source">
+ <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/>
+ <retention limit="hours(48)" action="delete"/>
+ <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+ <sla slaLow="hours(3)" slaHigh="hours(4)"/>
+ <export>
+ <target name="test-hsql-db" tableName="customer">
+ <load type="updateonly"/>
+ <fields>
+ <includes>
+ <field>id</field>
+ <field>name</field>
+ </includes>
+ </fields>
+ </target>
+ <arguments>
+ <argument name="--update-key" value="id"/>
+ </arguments>
+ </export>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location type="data" path="/projects/falcon/clicks"/>
+ <location type="stats" path="/projects/falcon/clicksStats"/>
+ <location type="meta" path="/projects/falcon/clicksMetaData"/>
+ </locations>
+
+ <ACL owner="testuser" group="group" permission="0x755"/>
+ <schema location="/schema/clicks" provider="protobuf"/>
+</feed>
http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/webapp/src/test/resources/feed-export-template6.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/feed-export-template6.xml b/webapp/src/test/resources/feed-export-template6.xml
index 0eb748b..0534257 100644
--- a/webapp/src/test/resources/feed-export-template6.xml
+++ b/webapp/src/test/resources/feed-export-template6.xml
@@ -31,12 +31,6 @@
<export>
<target name="##datasourcename##" tableName="simple_export">
<load type="allowinsert"/>
- <fields>
- <includes>
- <field>id</field>
- <field>name</field>
- </includes>
- </fields>
</target>
<arguments>
<argument name="--update-key" value="id"/>