You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ro...@apache.org on 2014/12/05 17:21:28 UTC
incubator-falcon git commit: FALCON-884 Add option to dump xmls
generated by falcon. Contributed by Raghav Kumar Gautam
Repository: incubator-falcon
Updated Branches:
refs/heads/master 56900a2b8 -> bf9ed92b3
FALCON-884 Add option to dump xmls generated by falcon. Contributed by Raghav Kumar Gautam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/bf9ed92b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/bf9ed92b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/bf9ed92b
Branch: refs/heads/master
Commit: bf9ed92b3a60a350d4589ff2a6f8485ab238d23e
Parents: 56900a2
Author: Ruslan Ostafiychuk <ro...@apache.org>
Authored: Fri Dec 5 18:16:51 2014 +0200
Committer: Ruslan Ostafiychuk <ro...@apache.org>
Committed: Fri Dec 5 18:16:51 2014 +0200
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 3 ++
falcon-regression/README.md | 6 +++
.../core/enumsAndConstants/MerlinConstants.java | 8 +++
.../falcon/regression/core/util/BundleUtil.java | 6 +--
.../falcon/regression/core/util/Config.java | 28 ++++++++---
.../falcon/regression/TestngListener.java | 52 +++++++++++++++++---
6 files changed, 86 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index a0a1a7d..ea9ac36 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,9 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-884 Add option to dump xmls generated by falcon (Raghav Kumar Gautam via Ruslan
+ Ostafiychuk)
+
FALCON-893 Add tests for replication to wasb filesystem
(Raghav Kumar Gautam and Ruslan Ostafiychuk via Arpit Gupta)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/README.md
----------------------------------------------------------------------
diff --git a/falcon-regression/README.md b/falcon-regression/README.md
index eca1ca0..33fd7cf 100644
--- a/falcon-regression/README.md
+++ b/falcon-regression/README.md
@@ -142,3 +142,9 @@ info and logs at the end of the test. This can be done by configuring Merlin.pro
log.capture.oozie.skip_info = false
log.capture.oozie.skip_log = true
log.capture.location = ../
+
+Dumping entities generated by falcon
+------------------------------------
+Add -Dmerlin.dump.staging to the maven command. For example:
+
+ mvn clean test -Phadoop-2 -Dmerlin.dump.staging=true
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
index 354137c..10cddf0 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
@@ -37,6 +37,14 @@ public final class MerlinConstants {
public static final boolean IS_SECURE =
"kerberos".equals(new Configuration().get("hadoop.security.authentication", "simple"));
+
+ /** Staging location to use in cluster xml. */
+ public static final String STAGING_LOCATION = Config.getProperty("merlin.staging.location",
+ "/tmp/falcon-regression-staging");
+ /** Working location to use in cluster xml. */
+ public static final String WORKING_LOCATION = Config.getProperty("merlin.working.location",
+ "/tmp/falcon-regression-working");
+
/** the user that is going to run tests. */
public static final String CURRENT_USER_NAME = System.getProperty("user.name");
/** keytab of current user. */
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
index f70b8f5..f52d563 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
@@ -112,13 +112,11 @@ public final class BundleUtil {
clusterMerlin.getLocations().getLocations().clear();
final Location staging = new Location();
staging.setName("staging");
- staging.setPath(Config.getProperty("merlin.staging.location",
- "/tmp/falcon-regression-staging"));
+ staging.setPath(MerlinConstants.STAGING_LOCATION);
clusterMerlin.getLocations().getLocations().add(staging);
final Location working = new Location();
working.setName("working");
- working.setPath(Config.getProperty("merlin.working.location",
- "/tmp/falcon-regression-working"));
+ working.setPath(MerlinConstants.WORKING_LOCATION);
clusterMerlin.getLocations().getLocations().add(working);
final String protectionPropName = "hadoop.rpc.protection";
final String protectionPropValue = Config.getProperty(protectionPropName);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
index 9172455..ba32d11 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
@@ -18,8 +18,11 @@
package org.apache.falcon.regression.core.util;
+import org.apache.commons.configuration.AbstractConfiguration;
+import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration.SystemConfiguration;
import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
import org.apache.log4j.Logger;
import org.testng.Assert;
@@ -31,20 +34,31 @@ public final class Config {
private static final String MERLIN_PROPERTIES = "Merlin.properties";
private static final Config INSTANCE = new Config(MERLIN_PROPERTIES);
- private PropertiesConfiguration confObj;
+ private AbstractConfiguration confObj;
private Config(String propFileName) {
try {
- LOGGER.info("Going to read properties from: " + propFileName);
- confObj = new PropertiesConfiguration(Config.class.getResource("/" + propFileName));
- //if changed configuration will be reloaded within 2 min
- final FileChangedReloadingStrategy reloadingStrategy = new FileChangedReloadingStrategy();
- reloadingStrategy.setRefreshDelay(2 * 60 * 1000);
- confObj.setReloadingStrategy(reloadingStrategy);
+ initConfig(propFileName);
} catch (ConfigurationException e) {
Assert.fail("Could not read properties because of exception: " + e);
}
}
+ private void initConfig(String propFileName) throws ConfigurationException {
+ CompositeConfiguration compositeConfiguration = new CompositeConfiguration();
+ LOGGER.info("Going to add properties from system properties.");
+ compositeConfiguration.addConfiguration(new SystemConfiguration());
+
+ LOGGER.info("Going to read properties from: " + propFileName);
+ final PropertiesConfiguration merlinConfig =
+ new PropertiesConfiguration(Config.class.getResource("/" + propFileName));
+ //if changed configuration will be reloaded within 2 min
+ final FileChangedReloadingStrategy reloadingStrategy = new FileChangedReloadingStrategy();
+ reloadingStrategy.setRefreshDelay(2 * 60 * 1000);
+ merlinConfig.setReloadingStrategy(reloadingStrategy);
+ compositeConfiguration.addConfiguration(merlinConfig);
+ this.confObj = compositeConfiguration;
+ }
+
public static String getProperty(String key) {
return INSTANCE.confObj.getString(key);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
index 645d63b..e3aac1d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
@@ -19,13 +19,17 @@
package org.apache.falcon.regression;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.util.Config;
import org.apache.falcon.regression.core.util.LogUtil;
import org.apache.falcon.regression.core.util.OSUtil;
+import org.apache.falcon.regression.core.util.TimeUtil;
import org.apache.falcon.regression.testHelper.BaseUITestClass;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import org.openqa.selenium.OutputType;
@@ -38,6 +42,7 @@ import org.testng.ITestResult;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Date;
/**
* Testng listener class. This is useful for things that are applicable to all the tests as well
@@ -47,6 +52,8 @@ public class TestngListener implements ITestListener, IExecutionListener {
private static final Logger LOGGER = Logger.getLogger(TestngListener.class);
private final String hr = StringUtils.repeat("-", 100);
+ private enum RunResult {SUCCESS, FAILED, SKIPPED, TestFailedButWithinSuccessPercentage }
+
@Override
public void onTestStart(ITestResult result) {
LOGGER.info(hr);
@@ -56,7 +63,12 @@ public class TestngListener implements ITestListener, IExecutionListener {
NDC.push(result.getName());
}
- private void logEndOfTest(ITestResult result, String outcome) {
+ private void endOfTestHook(ITestResult result, RunResult outcome) {
+ try {
+ dumpFalconStore(result);
+ } catch (Exception e) {
+ LOGGER.info("Dumping of falcon store failed: " + e);
+ }
LOGGER.info(
String.format("Testing going to end for: %s.%s(%s) %s", result.getTestClass().getName(),
result.getName(), Arrays.toString(result.getParameters()), outcome));
@@ -64,14 +76,42 @@ public class TestngListener implements ITestListener, IExecutionListener {
LOGGER.info(hr);
}
+ private void dumpFalconStore(ITestResult result) throws IOException {
+ if (Config.getBoolean("merlin.dump.staging", false)) {
+ final String[] serverNames = Config.getStringArray("servers");
+ for (final String serverName : serverNames) {
+ final ColoHelper coloHelper = new ColoHelper(serverName.trim());
+ final FileSystem clusterFs = coloHelper.getClusterHelper().getHadoopFS();
+ final String fileNameTemp = StringUtils.join(
+ new String[]{serverName,
+ result.getName(),
+ Arrays.toString(result.getParameters()),
+ TimeUtil.dateToOozieDate(new Date()),
+ },
+ "-");
+ final String localFileName = fileNameTemp.replaceAll(":", "-");
+ LOGGER.info("Dumping staging contents to: " + fileNameTemp);
+ clusterFs.copyToLocalFile(false, new Path(MerlinConstants.STAGING_LOCATION),
+ new Path(localFileName));
+ }
+ }
+ }
+
@Override
public void onTestSuccess(ITestResult result) {
- logEndOfTest(result, "SUCCESS");
+ endOfTestHook(result, RunResult.SUCCESS);
}
@Override
public void onTestFailure(ITestResult result) {
- logEndOfTest(result, "FAILED");
+ endOfTestHook(result, RunResult.FAILED);
+ takeScreenShot(result);
+
+ LOGGER.info(ExceptionUtils.getStackTrace(result.getThrowable()));
+ LOGGER.info(hr);
+ }
+
+ private void takeScreenShot(ITestResult result) {
if (BaseUITestClass.getDriver() != null) {
byte[] scrFile =
((TakesScreenshot)BaseUITestClass.getDriver()).getScreenshotAs(OutputType.BYTES);
@@ -90,12 +130,12 @@ public class TestngListener implements ITestListener, IExecutionListener {
@Override
public void onTestSkipped(ITestResult result) {
- logEndOfTest(result, "SKIPPED");
+ endOfTestHook(result, RunResult.SKIPPED);
}
@Override
public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
- logEndOfTest(result, "TestFailedButWithinSuccessPercentage");
+ endOfTestHook(result, RunResult.TestFailedButWithinSuccessPercentage);
}
@Override