You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sa...@apache.org on 2014/08/08 06:37:55 UTC
[1/2] git commit: FALCON 575 add test cases for dry run feature in
falcon-regression and minor fix
Repository: incubator-falcon
Updated Branches:
refs/heads/master 4bc578708 -> 5cb6afafa
FALCON 575 add test cases for dry run feature in falcon-regression and minor fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/993534d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/993534d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/993534d3
Branch: refs/heads/master
Commit: 993534d3ff2a14b03ba295532007a983c0651d7d
Parents: 4bc5787
Author: Samarth Gupta <sa...@inmobi.com>
Authored: Fri Aug 8 10:04:57 2014 +0530
Committer: Samarth Gupta <sa...@inmobi.com>
Committed: Fri Aug 8 10:04:57 2014 +0530
----------------------------------------------------------------------
.../regression/ProcessInstanceResumeTest.java | 13 +-
.../regression/prism/EntityDryRunTest.java | 150 +++++++++++++++++++
2 files changed, 156 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/993534d3/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
index 72457de..6bcb24a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
@@ -65,7 +65,6 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
Bundle b = BundleUtil.readELBundle();
b = new Bundle(b, cluster);
- b = new Bundle(b, cluster);
String startDate = "2010-01-01T23:20Z";
String endDate = "2010-01-02T01:40Z";
b.setInputFeedDataPath(feedInputPath);
@@ -169,7 +168,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
bundles[0].submitFeedsScheduleProcess(prism);
prism.getProcessHelper()
.getProcessInstanceSuspend(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:05Z&end=2010-01-02T01:20Z");
+ "?start=2010-01-02T01:05Z&end=2010-01-02T01:21Z");
TimeUtil.sleepSeconds(15);
InstancesResult result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
@@ -178,7 +177,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
prism.getProcessHelper()
.getProcessInstanceResume(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:05Z&end=2010-01-02T01:20Z");
+ "?start=2010-01-02T01:05Z&end=2010-01-02T01:21Z");
result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
"?start=2010-01-02T01:00Z&end=2010-01-02T01:26Z");
@@ -314,14 +313,14 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
"?start=2010-01-02T01:25Z");
InstancesResult result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:00Z&end=2010-01-02T01:25Z");
+ "?start=2010-01-02T01:00Z&end=2010-01-02T01:26Z");
InstanceUtil.validateResponse(result, 6, 5, 1, 0, 0);
prism.getProcessHelper()
.getProcessInstanceResume(Util.readEntityName(bundles[0].getProcessData()),
"?start=2010-01-02T01:25Z");
result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:00Z&end=2010-01-02T01:25Z");
+ "?start=2010-01-02T01:00Z&end=2010-01-02T01:26Z");
InstanceUtil.validateResponse(result, 6, 6, 0, 0, 0);
}
@@ -341,7 +340,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
TimeUtil.sleepSeconds(15);
prism.getProcessHelper()
.getProcessInstanceSuspend(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:05Z&end=2010-01-02T01:20Z");
+ "?start=2010-01-02T01:05Z&end=2010-01-02T01:21Z");
TimeUtil.sleepSeconds(15);
InstancesResult result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
@@ -350,7 +349,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass {
prism.getProcessHelper()
.getProcessInstanceResume(Util.readEntityName(bundles[0].getProcessData()),
- "?start=2010-01-02T01:05Z&end=2010-01-02T01:20Z");
+ "?start=2010-01-02T01:05Z&end=2010-01-02T01:21Z");
result = prism.getProcessHelper()
.getProcessInstanceStatus(Util.readEntityName(bundles[0].getProcessData()),
"?start=2010-01-02T01:00Z&end=2010-01-02T01:26Z");
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/993534d3/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
new file mode 100644
index 0000000..9bed6f2
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression.prism;
+
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.regression.core.bundle.Bundle;
+import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.response.ServiceResponse;
+import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.BundleUtil;
+import org.apache.falcon.regression.core.util.HadoopUtil;
+import org.apache.falcon.regression.core.util.OSUtil;
+import org.apache.falcon.regression.core.util.OozieUtil;
+import org.apache.falcon.regression.core.util.TimeUtil;
+import org.apache.falcon.regression.core.util.Util;
+import org.apache.falcon.regression.testHelper.BaseTestClass;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.xml.bind.JAXBException;
+import java.lang.reflect.Method;
+
+/*
+test cases for https://issues.apache.org/jira/browse/FALCON-353
+ */
+public class EntityDryRunTest extends BaseTestClass {
+
+ private ColoHelper cluster = servers.get(0);
+ private FileSystem clusterFS = serverFS.get(0);
+ private String baseTestHDFSDir = baseHDFSDir + "/EntityDryRunTest";
+ private String feedInputPath = baseTestHDFSDir +
+ "/input/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}";
+ private String feedOutputPath =
+ baseTestHDFSDir + "/output-data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}";
+ private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
+ private static final Logger LOGGER = Logger.getLogger(EntityDryRunTest.class);
+
+ @BeforeClass(alwaysRun = true)
+ public void createTestData() throws Exception {
+ LOGGER.info("in @BeforeClass");
+ HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
+ Bundle b = BundleUtil.readELBundle();
+ b = new Bundle(b, cluster);
+ b.setInputFeedDataPath(feedInputPath);
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setup(Method method) throws Exception {
+ LOGGER.info("setup " + method.getName());
+ bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = new Bundle(bundles[0], cluster);
+ bundles[0].generateUniqueBundle();
+ bundles[0].setInputFeedDataPath(feedInputPath);
+ bundles[0].setOutputFeedLocationData(feedOutputPath);
+ bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void tearDown(Method method) {
+ LOGGER.info("tearDown " + method.getName());
+ removeBundles();
+ }
+
+ /**
+ *
+ * tries to submit process with invalid el exp
+ */
+ @Test(groups = {"singleCluster"})
+ public void testDryRunFailureScheduleProcess() throws Exception {
+ bundles[0].setProcessProperty("EntityDryRunTestProp", "${coord:someEL(1)");
+ bundles[0].submitProcess(true);
+ ServiceResponse response = prism.getProcessHelper()
+ .schedule(Util.URLS.SCHEDULE_URL, bundles[0].getProcessData());
+ validate(response);
+ }
+
+ /**
+ *
+ * tries to update process with invalid EL exp
+ */
+ @Test(groups = {"singleCluster"})
+ public void testDryRunFailureUpdateProcess() throws Exception {
+ bundles[0].setProcessValidity(TimeUtil.getTimeWrtSystemTime(-10), TimeUtil.getTimeWrtSystemTime(100));
+ bundles[0].submitAndScheduleProcess();
+ bundles[0].setProcessProperty("EntityDryRunTestProp", "${coord:someEL(1)");
+ ServiceResponse response = prism.getProcessHelper().update(bundles[0].getProcessData(),
+ bundles[0].getProcessData(), TimeUtil.getTimeWrtSystemTime(5), null);
+ validate(response);
+ Assert.assertEquals(OozieUtil.getNumberOfBundle(cluster, EntityType.PROCESS, bundles[0].getProcessName()), 1,
+ "more than one bundle found after failed update request");
+ }
+
+ /**
+ * tries to submit feed with invalied EL exp
+ *
+ */
+ @Test(groups = {"singleCluster"})
+ public void testDryRunFailureScheduleFeed() throws Exception {
+ String feed = bundles[0].getInputFeedFromBundle();
+ feed = Util.setFeedProperty(feed, "EntityDryRunTestProp", "${coord:someEL(1)");
+ bundles[0].submitClusters(prism);
+ ServiceResponse response = prism.getFeedHelper().submitAndSchedule(Util.URLS.SUBMIT_AND_SCHEDULE_URL, feed);
+ validate(response);
+ }
+
+ /**
+ *
+ * tries to update feed with invalid el exp
+ */
+ @Test(groups = {"singleCluster"})
+ public void testDryRunFailureUpdateFeed() throws Exception {
+ bundles[0].submitClusters(prism);
+ String feed = bundles[0].getInputFeedFromBundle();
+ ServiceResponse response = prism.getFeedHelper().submitAndSchedule(Util.URLS.SUBMIT_AND_SCHEDULE_URL, feed);
+ AssertUtil.assertSucceeded(response);
+ feed = Util.setFeedProperty(feed, "EntityDryRunTestProp", "${coord:someEL(1)");
+ response = prism.getFeedHelper().update(feed, feed);
+ validate(response);
+ Assert.assertEquals(OozieUtil.getNumberOfBundle(cluster, EntityType.FEED, Util.readEntityName(feed)), 1,
+ "more than one bundle found after failed update request");
+ }
+
+ private void validate(ServiceResponse response) throws JAXBException {
+ AssertUtil.assertFailed(response);
+ Assert.assertTrue(response.getMessage().contains("org.apache.falcon.FalconException: AUTHENTICATION : E1004 :" +
+ " E1004: Expression language evaluation error, Unable to evaluate :${coord:someEL(1)"),
+ "Correct response was not present in process / feed schedule");
+ }
+}
[2/2] git commit: additional patch for falcon 564 fix pom files to
make falcon-regression a module and minor update
Posted by sa...@apache.org.
additional patch for falcon 564 fix pom files to make falcon-regression a module and minor update
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/5cb6afaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/5cb6afaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/5cb6afaf
Branch: refs/heads/master
Commit: 5cb6afafac20b5ee32a5e6946b51998418894961
Parents: 993534d
Author: Samarth Gupta <sa...@inmobi.com>
Authored: Fri Aug 8 10:07:03 2014 +0530
Committer: Samarth Gupta <sa...@inmobi.com>
Committed: Fri Aug 8 10:07:03 2014 +0530
----------------------------------------------------------------------
.../org/apache/falcon/regression/core/bundle/Bundle.java | 8 ++++++--
falcon-regression/pom.xml | 6 ++++++
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5cb6afaf/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 5be1730..57c7292 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -77,7 +77,7 @@ public class Bundle {
private List<String> dataSets;
private String processData;
- public void submitFeed() throws Exception {
+ public void submitFeed() throws URISyntaxException, IOException, AuthenticationException, JAXBException {
submitClusters(prismHelper);
AssertUtil.assertSucceeded(
@@ -110,12 +110,16 @@ public class Bundle {
public ServiceResponse submitProcess(boolean shouldSucceed) throws JAXBException,
IOException, URISyntaxException, AuthenticationException {
- submitAndScheduleAllFeeds();
+ submitClusters(prismHelper);
+ submitFeeds(prismHelper);
ServiceResponse r = prismHelper.getProcessHelper().submitEntity(URLS.SUBMIT_URL,
processData);
if (shouldSucceed) {
AssertUtil.assertSucceeded(r);
}
+ else {
+ AssertUtil.assertFailed(r);
+ }
return r;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5cb6afaf/falcon-regression/pom.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/pom.xml b/falcon-regression/pom.xml
index ad42dcf..e5bb31a 100644
--- a/falcon-regression/pom.xml
+++ b/falcon-regression/pom.xml
@@ -39,11 +39,17 @@
<module>merlin</module>
</modules>
+ <properties>
+ <oozie.version>3.2.2</oozie.version>
+ <hive.version>0.13.1</hive.version>
+ </properties>
+
<profiles>
<profile>
<id>hadoop-1</id>
<properties>
<hadoop1.version>0.20.2-cdh3u3</hadoop1.version>
+ <hive.version>0.12.0</hive.version>
</properties>
<dependencyManagement>
<dependencies>