You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pi...@apache.org on 2016/02/02 16:04:47 UTC
falcon git commit: FALCON-1766 Add CLI metrics check for HiveDR,
HDFS and feed replication. Contributed by Paul Isaychuk
Repository: falcon
Updated Branches:
refs/heads/master b5c0b6139 -> 4a76b11be
FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication. Contributed by Paul Isaychuk
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4a76b11b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4a76b11b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4a76b11b
Branch: refs/heads/master
Commit: 4a76b11be2a6f7831929831d91ecc5ac46e25260
Parents: b5c0b61
Author: Paul Isaychuk <pi...@apache.org>
Authored: Thu Jan 21 17:03:55 2016 +0200
Committer: Paul Isaychuk <pi...@apache.org>
Committed: Tue Feb 2 15:07:18 2016 +0200
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 +
.../core/helpers/FalconClientBuilder.java | 13 ++++
.../helpers/entity/AbstractEntityHelper.java | 12 +++
.../falcon/regression/core/util/AssertUtil.java | 42 +++++++++-
.../falcon/regression/FeedReplicationTest.java | 30 ++++---
.../regression/hive/dr/HdfsRecipeTest.java | 5 ++
.../falcon/regression/hive/dr/HiveDRTest.java | 5 ++
.../HdfsRecipe/hdfs-replication-template.xml | 44 +++++++++++
.../HdfsRecipe/hdfs-replication-workflow.xml | 82 ++++++++++++++++++++
.../HdfsRecipe/hdfs-replication.properties | 79 +++++++++++++++++++
.../hive-disaster-recovery-template.xml | 44 -----------
.../hive-disaster-recovery-workflow.xml | 82 --------------------
.../hive-disaster-recovery.properties | 79 -------------------
.../hive-disaster-recovery-workflow.xml | 2 +
14 files changed, 303 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index e6664f3..ccd9ceb 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication (Paul Isaychuk)
+
FALCON-1777 Add regression for HDFS replication (recipe) (Paul Isaychuk)
FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal)
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
index 2e6fca9..fcbb7f9 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
@@ -139,6 +139,19 @@ public final class FalconClientBuilder implements Builder<CommandLine> {
}
/**
+ * Create get metrics command.
+ * @param entityType type of the entity
+ * @param entityName name of the entity to be deleted
+ * @return this
+ */
+ public FalconClientBuilder getMetricsCommand(String entityType, String entityName) {
+ addArg("metadata").addArg("-list");
+ addArg("-type").addArg("replication_metrics");
+ addArg("-" + entityType).addArg(entityName);
+ return this;
+ }
+
+ /**
* Create submit command.
* @param entityType type of the entity
* @param fileName file containing the entity to be submitted
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index e406cae..29c97b2 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -593,6 +593,18 @@ public abstract class AbstractEntityHelper {
}
/**
+ * Get CLI metrics for recipe based process or feed replication.
+ * @param entityName
+ * @return
+ */
+ public ExecResult getCLIMetrics(String entityName) {
+ LOGGER.info("Getting CLI metrics for " + getEntityType()+ " " + entityName);
+ final CommandLine commandLine = FalconClientBuilder.getBuilder()
+ .getMetricsCommand(getEntityType(), entityName).build();
+ return ExecUtil.executeCommand(commandLine);
+ }
+
+ /**
* Delete an entity through falcon client.
* @param entityStr string of the entity to be submitted
* @throws IOException
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
index 9d3b802..0af0c1e 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
@@ -33,6 +33,8 @@ import org.apache.log4j.Logger;
import org.apache.oozie.client.Job;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
+import org.json.JSONArray;
+import org.json.JSONObject;
import org.testng.Assert;
import javax.xml.bind.JAXBException;
@@ -52,6 +54,38 @@ public final class AssertUtil {
private static final Logger LOGGER = Logger.getLogger(AssertUtil.class);
/**
+ * Asserts correctness of CLI metrics for recipe based process or feed replication.
+ * @param execResult CLI metrics exec result to be checked
+ * @param entityName name of recipe process or replication feed
+ * @param instanceNum expected number of process/feed instances in metrics output
+ * @param withData is data expected to be replicated
+ * @throws Exception
+ */
+ public static void assertCLIMetrics(ExecResult execResult, String entityName, int instanceNum, boolean withData)
+ throws Exception {
+ String output = execResult.getOutput();
+ Assert.assertTrue(StringUtils.isNotBlank(output), "Exec result output is blank.");
+ JSONObject jsonObject = new JSONObject(output);
+ int totalSize = jsonObject.getInt("totalSize");
+ Assert.assertEquals(totalSize, instanceNum);
+ JSONArray array = jsonObject.getJSONArray("results");
+ for (int i = 0; i < array.length(); i++) {
+ String name = array.getJSONObject(i).getString("name");
+ Assert.assertTrue(name.contains(entityName));
+ int timeTaken = array.getJSONObject(i).getInt("TIMETAKEN");
+ Assert.assertTrue(timeTaken > 0, "TIMETAKEN metric should be greater then zero.");
+ int bytescopied = array.getJSONObject(i).getInt("BYTESCOPIED");
+ Assert.assertTrue(bytescopied >= 0, "BYTESCOPIED metric should be greater or equal to zero.");
+ int copy = array.getJSONObject(i).getInt("COPY");
+ if (withData) {
+ Assert.assertTrue(copy > 0, "COPY metric should be greater then zero.");
+ } else {
+ Assert.assertEquals(copy, 0, "COPY metric should be equal to zero as data was absent.");
+ }
+ }
+ }
+
+ /**
* Checks that any path in list doesn't contains a string.
*
* @param paths list of paths
@@ -96,7 +130,7 @@ public final class AssertUtil {
LOGGER.info("elements = " + elements);
}
Assert.assertEquals(elements.size(), expectedSize,
- "Size of expected and actual list don't match.");
+ "Size of expected and actual list don't match.");
}
/**
@@ -218,7 +252,7 @@ public final class AssertUtil {
Assert.assertFalse(execResult.hasSuceeded(),
"Unexpectedly succeeded execResult: " + execResult);
Assert.assertTrue((execResult.getError() + execResult.getOutput()).contains(expectedMessage),
- "Expected error: " + expectedMessage + " in execResult: " + execResult);
+ "Expected error: " + expectedMessage + " in execResult: " + execResult);
}
/**
@@ -258,7 +292,7 @@ public final class AssertUtil {
*/
public static void assertFailed(ServiceResponse response) throws JAXBException {
Assert.assertNotEquals(response.getMessage(), "null",
- "response message should not be null");
+ "response message should not be null");
Assert.assertEquals(Util.parseResponse(response).getStatus(), APIResult.Status.FAILED);
Assert.assertEquals(response.getCode(), 400);
@@ -271,7 +305,7 @@ public final class AssertUtil {
*/
public static void assertFailed(APIResult response) {
Assert.assertNotEquals(response.getMessage(), "null",
- "response message should not be null");
+ "response message should not be null");
Assert.assertEquals(response.getStatus(), APIResult.Status.FAILED,
"Status should be FAILED. Message: " + response.getMessage());
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index 6728edf..a936aa1 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -24,6 +24,7 @@ import org.apache.falcon.entity.v0.feed.ClusterType;
import org.apache.falcon.regression.Entities.FeedMerlin;
import org.apache.falcon.regression.core.bundle.Bundle;
import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -36,11 +37,9 @@ import org.apache.falcon.regression.testHelper.BaseTestClass;
import org.apache.falcon.resource.InstancesResult;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.log4j.Logger;
import org.apache.oozie.client.CoordinatorAction;
import org.apache.oozie.client.OozieClient;
-import org.apache.oozie.client.OozieClientException;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
@@ -53,7 +52,6 @@ import org.testng.annotations.Test;
import javax.xml.bind.JAXBException;
import java.io.IOException;
-import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -133,6 +131,7 @@ public class FeedReplicationTest extends BaseTestClass {
.withClusterType(ClusterType.TARGET)
.withDataLocation(targetDataLocation)
.build());
+ feed.withProperty("job.counter", "true");
//submit and schedule feed
LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -178,7 +177,10 @@ public class FeedReplicationTest extends BaseTestClass {
Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true);
AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
- cluster2FS, "feed", "Success logs are not present");
+ cluster2FS, "feed", "Success logs are not present");
+
+ ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+ AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
}
/**
@@ -222,6 +224,7 @@ public class FeedReplicationTest extends BaseTestClass {
.withClusterType(ClusterType.TARGET)
.withDataLocation(targetDataLocation)
.build());
+ feed.withProperty("job.counter", "true");
//submit and schedule feed
LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -278,7 +281,10 @@ public class FeedReplicationTest extends BaseTestClass {
Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster3FS, toTarget, ""), true);
AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
- cluster2FS, "feed", "Success logs are not present");
+ cluster2FS, "feed", "Success logs are not present");
+
+ ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+ AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
}
/**
@@ -322,6 +328,7 @@ public class FeedReplicationTest extends BaseTestClass {
.withClusterType(ClusterType.TARGET)
.withDataLocation(targetDataLocation)
.build());
+ feed.withProperty("job.counter", "true");
//submit and schedule feed
LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -384,7 +391,10 @@ public class FeedReplicationTest extends BaseTestClass {
Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, availabilityFlagName), true);
AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
- cluster2FS, "feed", "Success logs are not present");
+ cluster2FS, "feed", "Success logs are not present");
+
+ ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+ AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
}
/**
@@ -392,9 +402,7 @@ public class FeedReplicationTest extends BaseTestClass {
* Check that new DistCp options are allowed.
*/
@Test
- public void testNewDistCpOptions()
- throws URISyntaxException, AuthenticationException, InterruptedException, IOException, JAXBException,
- OozieClientException {
+ public void testNewDistCpOptions() throws Exception {
Bundle.submitCluster(bundles[0], bundles[1]);
String startTime = TimeUtil.getTimeWrtSystemTime(0);
String endTime = TimeUtil.addMinsToTime(startTime, 5);
@@ -422,6 +430,7 @@ public class FeedReplicationTest extends BaseTestClass {
.withClusterType(ClusterType.TARGET)
.withDataLocation(targetDataLocation)
.build());
+ feed.withProperty("job.counter", "true");
//add custom properties to feed
HashMap<String, String> propMap = new HashMap<>();
@@ -485,6 +494,9 @@ public class FeedReplicationTest extends BaseTestClass {
List<Path> finalFiles = HadoopUtil.getAllFilesRecursivelyHDFS(cluster2FS, new Path(targetPath));
Assert.assertEquals(finalFiles.size(), 2, "Only replicated files should be present on target "
+ "because of 'removeDeletedFiles' distCp property.");
+
+ ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+ AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, true);
}
/**
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
index 05b9cf4..4a2d913 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
@@ -25,6 +25,7 @@ import org.apache.falcon.regression.Entities.ClusterMerlin;
import org.apache.falcon.regression.Entities.RecipeMerlin;
import org.apache.falcon.regression.core.bundle.Bundle;
import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -111,6 +112,10 @@ public class HdfsRecipeTest extends BaseTestClass {
.getAllFilesRecursivelyHDFS(clusterFS2, new Path(targetDataLocation));
AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData);
+
+ //particular check for https://issues.apache.org/jira/browse/FALCON-1643
+ ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(hdfsRecipe.getName());
+ AssertUtil.assertCLIMetrics(execResult, hdfsRecipe.getName(), 1, true);
}
@AfterMethod(alwaysRun = true)
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
index 7660e04..4dab0db 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
@@ -26,6 +26,8 @@ import org.apache.falcon.regression.Entities.RecipeMerlin;
import org.apache.falcon.regression.core.bundle.Bundle;
import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
+import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.supportClasses.NotifyingAssert;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HiveAssert;
@@ -164,6 +166,9 @@ public class HiveDRTest extends BaseTestClass {
HiveAssert.assertTableEqual(cluster, clusterHC.getTable(DB_NAME, tblName),
cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)
).assertAll();
+
+ ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(recipeMerlin.getName());
+ AssertUtil.assertCLIMetrics(execResult, recipeMerlin.getName(), 1, true);
}
@Test
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
new file mode 100644
index 0000000..46546ce
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <!-- source -->
+ <cluster name="##falcon.recipe.cluster.name##">
+ <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
+ </cluster>
+ </clusters>
+
+ <tags>_falcon_mirroring_type=HDFS</tags>
+
+ <parallel>1</parallel>
+ <!-- Dir replication needs to run only once to catch up -->
+ <order>LAST_ONLY</order>
+ <frequency>##falcon.recipe.frequency##</frequency>
+ <timezone>UTC</timezone>
+
+ <properties>
+ <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+ </properties>
+
+ <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
+ <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
+ <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
+ <ACL/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
new file mode 100644
index 0000000..aa820d0
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
@@ -0,0 +1,82 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'>
+ <start to='dr-replication'/>
+ <!-- Replication action -->
+ <action name="dr-replication">
+ <java>
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <configuration>
+ <property> <!-- hadoop 2 parameter -->
+ <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ <property>
+ <name>oozie.launcher.mapred.job.priority</name>
+ <value>${jobPriority}</value>
+ </property>
+ <property>
+ <name>oozie.use.system.libpath</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>oozie.action.sharelib.for.java</name>
+ <value>distcp</value>
+ </property>
+ <property>
+ <name>oozie.launcher.oozie.libpath</name>
+ <value>${wf:conf("falcon.libpath")}</value>
+ </property>
+ <property>
+ <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
+ <value>${drSourceClusterFS},${drTargetClusterFS}</value>
+ </property>
+ </configuration>
+ <main-class>org.apache.falcon.replication.FeedReplicator</main-class>
+ <arg>-Dmapred.job.queue.name=${queueName}</arg>
+ <arg>-Dmapred.job.priority=${jobPriority}</arg>
+ <arg>-maxMaps</arg>
+ <arg>${distcpMaxMaps}</arg>
+ <arg>-mapBandwidth</arg>
+ <arg>${distcpMapBandwidth}</arg>
+ <arg>-sourcePaths</arg>
+ <arg>${drSourceDir}</arg>
+ <arg>-targetPath</arg>
+ <arg>${drTargetClusterFS}${drTargetDir}</arg>
+ <arg>-falconFeedStorageType</arg>
+ <arg>FILESYSTEM</arg>
+ <arg>-availabilityFlag</arg>
+ <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
+ <arg>-counterLogDir</arg>
+ <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
+ </java>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+ <kill name="fail">
+ <message>
+ Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
+ </message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
new file mode 100644
index 0000000..6c715f3
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##### NOTE: This is a TEMPLATE file which can be copied and edited
+
+##### Recipe properties
+##### Unique recipe job name
+falcon.recipe.name=hdfs-replication-monthly
+
+##### Workflow properties
+falcon.recipe.workflow.name=hdfs-dr-workflow
+# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
+falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
+
+##### Cluster properties
+
+# Cluster where job should run
+falcon.recipe.cluster.name=primaryCluster
+# Change the cluster hdfs write end point here. This is mandatory.
+falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
+# Change the cluster validity start time here
+falcon.recipe.cluster.validity.start=2015-03-13T00:00Z
+# Change the cluster validity end time here
+falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
+
+##### Scheduling properties
+# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months
+falcon.recipe.process.frequency=minutes(5)
+
+##### Retry policy properties
+
+falcon.recipe.retry.policy=periodic
+falcon.recipe.retry.delay=minutes(30)
+falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
+
+##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
+##### Uncomment to add tags
+#falcon.recipe.tags=
+
+##### ACL properties - Uncomment and change ACL if authorization is enabled
+
+#falcon.recipe.acl.owner=ambari-qa
+#falcon.recipe.acl.group=users
+#falcon.recipe.acl.permission=0x755
+#falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
+
+##### Custom Job properties
+
+# Specify multiple comma separated source directories
+drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input
+drSourceClusterFS=hdfs://240.0.0.10:8020
+drTargetDir=/user/falcon_qa/dr/test/backupCluster/input
+drTargetClusterFS=hdfs://240.0.0.11:8020
+
+# Change it to specify the maximum number of mappers for DistCP
+distcpMaxMaps=1
+# Change it to specify the bandwidth in MB for each mapper in DistCP
+distcpMapBandwidth=100
+
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
+
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
deleted file mode 100644
index 46546ce..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1">
- <clusters>
- <!-- source -->
- <cluster name="##falcon.recipe.cluster.name##">
- <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
- </cluster>
- </clusters>
-
- <tags>_falcon_mirroring_type=HDFS</tags>
-
- <parallel>1</parallel>
- <!-- Dir replication needs to run only once to catch up -->
- <order>LAST_ONLY</order>
- <frequency>##falcon.recipe.frequency##</frequency>
- <timezone>UTC</timezone>
-
- <properties>
- <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
- </properties>
-
- <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
- <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
- <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
- <ACL/>
-</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
deleted file mode 100644
index aa820d0..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
+++ /dev/null
@@ -1,82 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'>
- <start to='dr-replication'/>
- <!-- Replication action -->
- <action name="dr-replication">
- <java>
- <job-tracker>${jobTracker}</job-tracker>
- <name-node>${nameNode}</name-node>
- <configuration>
- <property> <!-- hadoop 2 parameter -->
- <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
- <value>true</value>
- </property>
- <property>
- <name>mapred.job.queue.name</name>
- <value>${queueName}</value>
- </property>
- <property>
- <name>oozie.launcher.mapred.job.priority</name>
- <value>${jobPriority}</value>
- </property>
- <property>
- <name>oozie.use.system.libpath</name>
- <value>true</value>
- </property>
- <property>
- <name>oozie.action.sharelib.for.java</name>
- <value>distcp</value>
- </property>
- <property>
- <name>oozie.launcher.oozie.libpath</name>
- <value>${wf:conf("falcon.libpath")}</value>
- </property>
- <property>
- <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
- <value>${drSourceClusterFS},${drTargetClusterFS}</value>
- </property>
- </configuration>
- <main-class>org.apache.falcon.replication.FeedReplicator</main-class>
- <arg>-Dmapred.job.queue.name=${queueName}</arg>
- <arg>-Dmapred.job.priority=${jobPriority}</arg>
- <arg>-maxMaps</arg>
- <arg>${distcpMaxMaps}</arg>
- <arg>-mapBandwidth</arg>
- <arg>${distcpMapBandwidth}</arg>
- <arg>-sourcePaths</arg>
- <arg>${drSourceDir}</arg>
- <arg>-targetPath</arg>
- <arg>${drTargetClusterFS}${drTargetDir}</arg>
- <arg>-falconFeedStorageType</arg>
- <arg>FILESYSTEM</arg>
- <arg>-availabilityFlag</arg>
- <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
- <arg>-counterLogDir</arg>
- <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
- </java>
- <ok to="end"/>
- <error to="fail"/>
- </action>
- <kill name="fail">
- <message>
- Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
- </message>
- </kill>
- <end name="end"/>
-</workflow-app>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
deleted file mode 100644
index 6c715f3..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##### NOTE: This is a TEMPLATE file which can be copied and edited
-
-##### Recipe properties
-##### Unique recipe job name
-falcon.recipe.name=hdfs-replication-monthly
-
-##### Workflow properties
-falcon.recipe.workflow.name=hdfs-dr-workflow
-# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
-
-##### Cluster properties
-
-# Cluster where job should run
-falcon.recipe.cluster.name=primaryCluster
-# Change the cluster hdfs write end point here. This is mandatory.
-falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
-# Change the cluster validity start time here
-falcon.recipe.cluster.validity.start=2015-03-13T00:00Z
-# Change the cluster validity end time here
-falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
-
-##### Scheduling properties
-# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months
-falcon.recipe.process.frequency=minutes(5)
-
-##### Retry policy properties
-
-falcon.recipe.retry.policy=periodic
-falcon.recipe.retry.delay=minutes(30)
-falcon.recipe.retry.attempts=3
-falcon.recipe.retry.onTimeout=false
-
-##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
-##### Uncomment to add tags
-#falcon.recipe.tags=
-
-##### ACL properties - Uncomment and change ACL if authorization is enabled
-
-#falcon.recipe.acl.owner=ambari-qa
-#falcon.recipe.acl.group=users
-#falcon.recipe.acl.permission=0x755
-#falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
-
-##### Custom Job properties
-
-# Specify multiple comma separated source directories
-drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input
-drSourceClusterFS=hdfs://240.0.0.10:8020
-drTargetDir=/user/falcon_qa/dr/test/backupCluster/input
-drTargetClusterFS=hdfs://240.0.0.11:8020
-
-# Change it to specify the maximum number of mappers for DistCP
-distcpMaxMaps=1
-# Change it to specify the bandwidth in MB for each mapper in DistCP
-distcpMapBandwidth=100
-
-##### Email Notification for Falcon instance completion
-falcon.recipe.notification.type=email
-falcon.recipe.notification.receivers=NA
-
http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
index 72d40a3..66f97c4 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
+++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
@@ -158,6 +158,8 @@
<arg>${drJobName}-${nominalTime}</arg>
<arg>-executionStage</arg>
<arg>export</arg>
+ <arg>-counterLogDir</arg>
+ <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
</java>
<ok to="import-dr-replication"/>
<error to="fail"/>