You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pr...@apache.org on 2016/01/11 11:39:16 UTC
falcon git commit: FALCON-1699 Test fixes for RetentionTest,
LineageApiTest, TouchAPIPrismAndServerTest,
FeedReplicationTest and few fortifications. Contributed by Paul Isaychuk.
Repository: falcon
Updated Branches:
refs/heads/master 4656f692a -> e9e0c7b7c
FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications. Contributed by Paul Isaychuk.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e9e0c7b7
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e9e0c7b7
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e9e0c7b7
Branch: refs/heads/master
Commit: e9e0c7b7c0aed12e36c61f2cf8b4e67458fdfd06
Parents: 4656f69
Author: Pragya <mi...@gmail.com>
Authored: Mon Jan 11 10:30:06 2016 +0000
Committer: Pragya <mi...@gmail.com>
Committed: Mon Jan 11 10:30:06 2016 +0000
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 +
.../regression/core/response/lineage/Edge.java | 3 ++
.../regression/core/util/KerberosHelper.java | 9 ----
.../falcon/regression/core/util/OozieUtil.java | 52 +++++++++++++-------
.../org/apache/falcon/request/BaseRequest.java | 3 ++
.../regression/TouchAPIPrismAndServerTest.java | 1 +
.../regression/hcat/HCatRetentionTest.java | 9 +++-
.../lineage/ListFeedInstancesTest.java | 12 ++---
.../falcon/regression/prism/RetentionTest.java | 6 ++-
.../regression/security/FalconClientTest.java | 7 +--
10 files changed, 63 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index a4ae687..f2df91a 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal)
+
FALCON-1698 New tests for ProcessSetupTest, ClusterSetupTest, UI test fixes(Paul Isaychuk via Ajay Yadava)
FALCON-1700 Add new test cases to HiveDRTest(Paul Isaychuk & Murali Ramasami via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
index c1a7eb8..f1c408e 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
@@ -58,6 +58,9 @@ public class Edge extends GraphEntity {
@SerializedName("ninth")NINTH,
@SerializedName("tenth")TENTH,
@SerializedName("value")VALUE,
+ @SerializedName("_falcon_mirroring_type")MIRRORING_TYPE,
+ @SerializedName("specific")SPECIFIC,
+ @SerializedName("myTag1")MY_TAG
}
@SerializedName("_id")
private String id;
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
index c9f540f..9d028fa 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
@@ -18,7 +18,6 @@
package org.apache.falcon.regression.core.util;
-import org.apache.commons.exec.CommandLine;
import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
import org.apache.hadoop.security.UserGroupInformation;
@@ -42,14 +41,6 @@ public final class KerberosHelper {
getKeyTab(user));
}
- /**
- * Switches user in kerberos.
- */
- public static void initUserWithKeytab(String user){
- ExecUtil.executeCommand(new CommandLine("sudo").addArgument("kinit").addArgument(getPrincipal(user))
- .addArgument("-k").addArgument("-t").addArgument(getKeyTab(user)));
- }
-
private static String getKeyTab(String user) {
return MerlinConstants.getKeytabForUser(user);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
index ae96044..e73bc5d 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
@@ -502,29 +502,45 @@ public final class OozieUtil {
Assert.assertEquals(actualRun, attempts, "Rerun attempts did not match");
}
+ /**
+ * Try to find feed coordinators of given type.
+ */
+ public static int checkIfFeedCoordExist(OozieClient oozieClient,
+ String feedName, String coordType) throws OozieClientException {
+ return checkIfFeedCoordExist(oozieClient, feedName, coordType, 5);
+ }
+
+ /**
+ * Try to find feed coordinators of given type given number of times.
+ */
public static int checkIfFeedCoordExist(OozieClient oozieClient,
- String feedName, String coordType) throws OozieClientException {
+ String feedName, String coordType, int numberOfRetries) throws OozieClientException {
LOGGER.info("feedName: " + feedName);
- int numberOfCoord = 0;
- if (getBundles(oozieClient, feedName, EntityType.FEED).size() == 0) {
- return 0;
- }
- List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED);
- LOGGER.info("bundleIds: " + bundleIds);
-
- for (String aBundleId : bundleIds) {
- LOGGER.info("aBundleId: " + aBundleId);
- waitForCoordinatorJobCreation(oozieClient, aBundleId);
- List<CoordinatorJob> coords =
- getBundleCoordinators(oozieClient, aBundleId);
- LOGGER.info("coords: " + coords);
- for (CoordinatorJob coord : coords) {
- if (coord.getAppName().contains(coordType)) {
- numberOfCoord++;
+ for (int retryAttempt = 0; retryAttempt < numberOfRetries; retryAttempt++) {
+ int numberOfCoord = 0;
+ List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED);
+ if (bundleIds.size() == 0) {
+ TimeUtil.sleepSeconds(4);
+ continue;
+ }
+ LOGGER.info("bundleIds: " + bundleIds);
+ for (String aBundleId : bundleIds) {
+ LOGGER.info("aBundleId: " + aBundleId);
+ waitForCoordinatorJobCreation(oozieClient, aBundleId);
+ List<CoordinatorJob> coords = getBundleCoordinators(oozieClient, aBundleId);
+ LOGGER.info("coords: " + coords);
+ for (CoordinatorJob coord : coords) {
+ if (coord.getAppName().contains(coordType)) {
+ numberOfCoord++;
+ }
}
}
+ if (numberOfCoord > 0) {
+ return numberOfCoord;
+ }
+ TimeUtil.sleepSeconds(4);
}
- return numberOfCoord;
+ return 0;
}
/**
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
index e5430eb..60eeabd 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
@@ -45,6 +45,7 @@ import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.BasicClientConnectionManager;
import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import javax.net.ssl.SSLContext;
@@ -198,6 +199,8 @@ public class BaseRequest {
LOGGER.info(String.format("Request Header: Name=%s Value=%s", header.getName(),
header.getValue()));
}
+ // Must call this to release the connection
+ EntityUtils.consume(response.getEntity());
response = client.execute(target, request);
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
index 1bffe9a..25cf16f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
@@ -105,6 +105,7 @@ public class TouchAPIPrismAndServerTest extends BaseTestClass {
// via server
oldbundleId = bundleId;
+ InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 1);
coordId = OozieUtil.getLatestCoordinatorID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS);
response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData());
bundleId = OozieUtil.getLatestBundleID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS);
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index d639c21..49ed94a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -262,7 +262,14 @@ public class HCatRetentionTest extends BaseTestClass {
//Each HCat partition maps to a directory, not to a file
HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName,
tableName, dataFolder, ptn).build();
- client.addPartition(addPtn);
+ try {
+ client.addPartition(addPtn);
+ } catch (HCatException e) {
+ //occurrence of specific hive metastore bug
+ if (!e.getMessage().contains("AlreadyExistsException")) {
+ throw e;
+ }
+ }
ptn.clear();
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
index 17725ae..b3ca4f0 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
@@ -69,8 +69,8 @@ public class ListFeedInstancesTest extends BaseTestClass {
private String feedDataLocation = sourcePath + MINUTE_DATE_PATTERN;
private String targetPath = baseTestHDFSDir + "/target";
private String targetDataLocation = targetPath + MINUTE_DATE_PATTERN;
- private final String startTime = "2010-01-02T00:00Z";
- private final String endTime = "2010-01-02T00:57Z";
+ private final String startTime = "2015-01-02T00:00Z";
+ private final String endTime = "2015-01-02T00:57Z";
private String feedName;
@BeforeClass(alwaysRun = true)
@@ -210,7 +210,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
throws URISyntaxException, OozieClientException, JAXBException, AuthenticationException,
IOException, InterruptedException {
SoftAssert softAssert = new SoftAssert();
- //orderBy start time, check an order
+ //orderBy start time, check on order
InstancesResult r = prism.getFeedHelper().listInstances(feedName,
"orderBy=startTime&sortOrder=desc", null);
InstancesResult.Instance[] instances = r.getInstances();
@@ -223,7 +223,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
previousDate = (Date) current.clone();
}
}
- //orderBy status, check an order
+ //orderBy status, check on order
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc", null);
InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
@@ -235,7 +235,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
"Wrong order. Compared " + current + " and " + previousStatus + " statuses.");
previousStatus = current;
}
- //sort by endTime, check an order
+ //sort by endTime, check on order
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12&orderBy=endTime&sortOrder=desc", null);
instances = r.getInstances();
@@ -252,7 +252,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
}
/**
- * List instances through api using start/end parameters. Check a number of instances.
+ * Test the list feed instance api using start/end parameters. Check instances number.
*/
@Test
public void testFeedStartEnd()
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index 8f45d1c..018b83a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -54,6 +54,7 @@ import javax.jms.JMSException;
import javax.jms.MapMessage;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -181,10 +182,13 @@ public class RetentionTest extends BaseTestClass {
JmsMessageConsumer messageConsumer = new JmsMessageConsumer("FALCON." + feedName,
cluster.getClusterHelper().getActiveMQ());
messageConsumer.start();
- final DateTime currentTime = new DateTime(DateTimeZone.UTC);
String bundleId = OozieUtil.getBundles(clusterOC, feedName, EntityType.FEED).get(0);
List<String> workflows = OozieUtil.waitForRetentionWorkflowToSucceed(bundleId, clusterOC);
+
+ //get current time minus duration of last status check - to get actual time when eviction has started
+ final DateTime currentTime = new DateTime(new DateTime(DateTimeZone.UTC).toDate().getTime() - 10000);
+ LOGGER.info("Current time is " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(currentTime.toDate()));
LOGGER.info("workflows: " + workflows);
messageConsumer.interrupt();
Util.printMessageData(messageConsumer);
http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
index 73273f9..d11411b 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
@@ -25,7 +25,6 @@ import org.apache.falcon.regression.core.supportClasses.ExecResult;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
-import org.apache.falcon.regression.core.util.KerberosHelper;
import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.testHelper.BaseTestClass;
import org.apache.hadoop.fs.FileSystem;
@@ -81,13 +80,9 @@ public class FalconClientTest extends BaseTestClass {
* able to delete
* @throws Exception
*/
- @Test(enabled = true)
+ @Test(enabled = false)
public void badClusterDelete() throws Exception {
bundles[0].submitClusters(prism);
- //switch user
- if (MerlinConstants.IS_SECURE) {
- KerberosHelper.initUserWithKeytab(MerlinConstants.DIFFERENT_USER_NAME);
- }
final String clusterXml = bundles[0].getClusters().get(0);
final ExecResult execResult =
prism.getClusterHelper().clientDelete(clusterXml, MerlinConstants.DIFFERENT_USER_NAME);