You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/08/13 11:08:46 UTC
[02/12] falcon git commit: FALCON-1188 Falcon support for Hive
Replication. Contributed by Venkat Ranganathan.
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/src/main/assemblies/distributed-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
index 1e48c94..0b56572 100644
--- a/src/main/assemblies/distributed-package.xml
+++ b/src/main/assemblies/distributed-package.xml
@@ -21,6 +21,7 @@
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<formats>
<format>tar.gz</format>
+ <format>zip</format>
</formats>
<id>server</id>
<baseDirectory>falcon-distributed-${project.version}</baseDirectory>
@@ -126,6 +127,20 @@
<source>oozie-el-extensions/src/main/conf/oozie-site.xml</source>
<outputDirectory>oozie/conf</outputDirectory>
</file>
+
+ <file>
+ <source>addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
+
+ <file>
+ <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
+ <file>
+ <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
</files>
</assembly>
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index 682d52f..40e6b99 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -21,6 +21,7 @@
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<formats>
<format>tar.gz</format>
+ <format>zip</format>
</formats>
<id>bin</id>
<baseDirectory>falcon-${project.version}</baseDirectory>
@@ -115,6 +116,21 @@
</file>
<file>
+ <source>addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
+
+ <file>
+ <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
+
+ <file>
+ <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml</source>
+ <outputDirectory>data-mirroring/workflows</outputDirectory>
+ </file>
+
+ <file>
<source>webapp/target/falcon-webapp-${project.version}.war</source>
<outputDirectory>server/webapp</outputDirectory>
<destName>falcon.war</destName>
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/test-tools/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/pom.xml b/test-tools/hadoop-webapp/pom.xml
index cadc7f5..c122a89 100644
--- a/test-tools/hadoop-webapp/pom.xml
+++ b/test-tools/hadoop-webapp/pom.xml
@@ -53,6 +53,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>compile</scope>
</dependency>
@@ -71,12 +77,24 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>compile</scope>
</dependency>
@@ -192,6 +210,11 @@
<!-- Oozie dependencies -->
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-webhcat-java-client</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 063d42c..5a9e1da 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -45,7 +45,6 @@
<webResources>
<resource>
<directory>../falcon-ui/dist</directory>
- <targetPath>public</targetPath>
</resource>
<resource>
<directory>src/main/webapp/WEB-INF/distributed</directory>
@@ -70,6 +69,16 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>hivedr</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-hive-replication</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<dependencies>
@@ -113,7 +122,8 @@
<dependency>
<groupId>org.apache.falcon</groupId>
- <artifactId>falcon-replication</artifactId>
+ <artifactId>falcon-distcp-replication</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index 90acb59..1e0a096 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -44,7 +44,7 @@ import java.util.Properties;
*/
@Test(groups = {"exhaustive"})
public class FalconCLIIT {
- private static final String RECIPE_PROPERTIES_FILE_XML = "/process.properties";
+ private static final String RECIPE_PROPERTIES_FILE_XML = "/hdfs-replication.properties";
private InMemoryWriter stream = new InMemoryWriter(System.out);
private String recipePropertiesFilePath;
@@ -217,23 +217,23 @@ public class FalconCLIIT {
Assert.assertEquals(executeWithURL("entity -status -type process -name " + overlay.get("processName")), 0);
Assert.assertEquals(executeWithURL("entity -summary -type feed -cluster "+ overlay.get("cluster")
- + " -fields status,tags -start " + START_INSTANCE
- + " -filterBy TYPE:FEED -orderBy name -sortOrder asc "
- + " -offset 0 -numResults 1 -numInstances 5"), 0);
+ + " -fields status,tags -start " + START_INSTANCE
+ + " -filterBy TYPE:FEED -orderBy name -sortOrder asc "
+ + " -offset 0 -numResults 1 -numInstances 5"), 0);
Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
- + " -cluster " + overlay.get("cluster")
- + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
- + " -end " + SchemaHelper.getDateFormat().format(new Date())
- + " -filterBy TYPE:PROCESS -orderBy name -sortOrder desc "
- + " -offset 0 -numResults 1 -numInstances 7"), 0);
+ + " -cluster " + overlay.get("cluster")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())
+ + " -filterBy TYPE:PROCESS -orderBy name -sortOrder desc "
+ + " -offset 0 -numResults 1 -numInstances 7"), 0);
Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
- + " -cluster " + overlay.get("cluster")
- + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
- + " -end " + SchemaHelper.getDateFormat().format(new Date())
- + " -filterBy TYPE:PROCESS -orderBy name -sortOrder invalid "
- + " -offset 0 -numResults 1 -numInstances 7"), -1);
+ + " -cluster " + overlay.get("cluster")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())
+ + " -filterBy TYPE:PROCESS -orderBy name -sortOrder invalid "
+ + " -offset 0 -numResults 1 -numInstances 7"), -1);
// No start or end date
Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
@@ -329,9 +329,9 @@ public class FalconCLIIT {
+ " -orderBy startTime -sortOrder asc -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start " + SchemaHelper.getDateFormat().format(new Date())
- + " -orderBy INVALID -offset 0 -numResults 1"), -1);
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date())
+ + " -orderBy INVALID -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
@@ -339,14 +339,14 @@ public class FalconCLIIT {
+ " -orderBy startTime -sortOrder desc -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start " + startTimeString
- + " -orderBy startTime -sortOrder invalid -offset 0 -numResults 1"), -1);
+ + overlay.get("outputFeedName")
+ + " -start " + startTimeString
+ + " -orderBy startTime -sortOrder invalid -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start " + SchemaHelper.getDateFormat().format(new Date())
- + " -filterBy INVALID:FILTER -offset 0 -numResults 1"), -1);
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date())
+ + " -filterBy INVALID:FILTER -offset 0 -numResults 1"), -1);
// testcase : start str is older than entity schedule time.
Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction -name "
@@ -389,14 +389,14 @@ public class FalconCLIIT {
+ " -orderBy startTime -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -status -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())
- +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"), -1);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -list -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())
- +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults 1"), -1);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -status -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
+ " -start " + SchemaHelper.getDateFormat().format(new Date())
@@ -453,8 +453,8 @@ public class FalconCLIIT {
// No end date, should fail.
Assert.assertEquals(executeWithURL("instance -suspend -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
Assert.assertEquals(executeWithURL("instance -resume -type process -name "
+ overlay.get("processName")
@@ -479,13 +479,13 @@ public class FalconCLIIT {
OozieTestUtils.waitForProcessWFtoStart(context);
Assert.assertEquals(executeWithURL("instance -kill -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
// Fail due to no end date
Assert.assertEquals(executeWithURL("instance -kill -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
+ overlay.get("processName")
@@ -575,13 +575,13 @@ public class FalconCLIIT {
+ " -tags owner=producer@xyz.com,department=forecasting "
+ " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("entity -list -type process -fields status "
- + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1"), -1);
+ + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("entity -list -type process -fields INVALID "
- + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset 1 -numResults 1"), -1);
+ + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset 1 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("entity -list -type process -fields status "
- + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1 -numResults 1"), -1);
+ + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster")), 0);
@@ -678,26 +678,26 @@ public class FalconCLIIT {
OozieTestUtils.waitForProcessWFtoStart(context);
Assert.assertEquals(executeWithURL("instance -kill -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
Assert.assertEquals(executeWithURL("instance -kill -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())
- + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE), -1);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE), -1);
Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
Assert.assertEquals(executeWithURL("instance -rerun -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())
- + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
}
public void testInvalidCLIInstanceCommands() throws Exception {
@@ -712,7 +712,7 @@ public class FalconCLIIT {
+ " -end 2010-01-01T03:00Z"), -1);
Assert.assertEquals(executeWithURL("instance -kill -type process -name "
- + " -start 2010-01-01T01:00Z -end 2010-01-01T03:00Z"), -1);
+ + " -start 2010-01-01T01:00Z -end 2010-01-01T03:00Z"), -1);
}
public void testFalconURL() throws Exception {
@@ -734,12 +734,12 @@ public class FalconCLIIT {
submitTestFiles(context, overlay);
Assert.assertEquals(new FalconCLI().run(("entity -schedule -type feed -name "
- + overlay.get("outputFeedName") + " -url "
- + TestContext.BASE_URL).split("\\s+")), 0);
+ + overlay.get("outputFeedName") + " -url "
+ + TestContext.BASE_URL).split("\\s+")), 0);
Assert.assertEquals(new FalconCLI().run(("entity -schedule -type process -name "
- + overlay.get("processName")+ " -url "
- + TestContext.BASE_URL).split("\\s+")), 0);
+ + overlay.get("processName")+ " -url "
+ + TestContext.BASE_URL).split("\\s+")), 0);
}
public void testGetVersion() throws Exception {
@@ -766,52 +766,52 @@ public class FalconCLIIT {
Thread.sleep(500);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
Assert.assertEquals(executeWithURL("instance -logs -type feed -lifecycle eviction -name "
- + overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())), 0);
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())), 0);
// test filterBy, orderBy, offset, numResults
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
- + " -sortOrder asc -offset 0 -numResults 1"), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
+ + " -sortOrder asc -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED -orderBy starttime "
- + " -sortOrder asc -offset 0 -numResults 1"), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy starttime "
+ + " -sortOrder asc -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED -orderBy cluster "
- + " -sortOrder asc -offset 0 -numResults 1"), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy cluster "
+ + " -sortOrder asc -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:WAITING -orderBy startTime -offset 0 -numResults 1"), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:WAITING -orderBy startTime -offset 0 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
- + " -sortOrder invalid -offset 0 -numResults 1"), -1);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
+ + " -sortOrder invalid -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset 1 -numResults 1"), 0);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset 1 -numResults 1"), 0);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"), -1);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"), -1);
Assert.assertEquals(executeWithURL("instance -logs -type process -name "
- + overlay.get("processName")
- + " -start " + START_INSTANCE + " -end " + START_INSTANCE
- + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults 1"), -1);
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults 1"), -1);
}
@SuppressWarnings("ResultOfMethodCallIgnored")
@@ -819,7 +819,8 @@ public class FalconCLIIT {
public void testRecipeCommand() throws Exception {
recipeSetup();
try {
- Assert.assertEquals(executeWithURL("recipe -name " + "process"), 0);
+ Assert.assertEquals(executeWithURL("recipe -name " + "hdfs-replication"
+ + " -operation HDFS_REPLICATION"), 0);
} finally {
if (recipePropertiesFilePath != null) {
new File(recipePropertiesFilePath).delete();
@@ -836,12 +837,6 @@ public class FalconCLIIT {
overlay);
Assert.assertEquals(executeWithURL("entity -submit -type cluster -file " + filePath), 0);
context.setCluster(overlay.get("cluster"));
-
- filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay);
- Assert.assertEquals(executeWithURL("entity -submit -type feed -file " + filePath), 0);
-
- filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay);
- Assert.assertEquals(executeWithURL("entity -submit -type feed -file " + filePath), 0);
}
private void createPropertiesFile(TestContext context) throws Exception {
@@ -854,16 +849,14 @@ public class FalconCLIIT {
String resourcePath = FilenameUtils.getFullPathNoEndSeparator(wfFile);
String libPath = TestContext.getTempFile("target/lib", "recipe", ".jar").getAbsolutePath();
- File file = new File(resourcePath, "process.properties");
+ File file = new File(resourcePath, "hdfs-replication.properties");
OutputStream out = new FileOutputStream(file);
- props.setProperty("falcon.recipe.processName", context.getProcessName());
- props.setProperty("falcon.recipe.src.cluster.name", context.getClusterName());
- props.setProperty("falcon.recipe.processEndDate", context.getProcessEndTime());
- props.setProperty("falcon.recipe.inputFeedName", context.getInputFeedName());
- props.setProperty("falcon.recipe.outputFeedName", context.getOutputFeedName());
+ props.setProperty("falcon.recipe.name", context.getProcessName());
+ props.setProperty("falcon.recipe.cluster.name", context.getClusterName());
+ props.setProperty("falcon.recipe.cluster.validity.end", context.getProcessEndTime());
props.setProperty("falcon.recipe.workflow.path", TestContext.class.getResource("/fs-workflow.xml").getPath());
props.setProperty("falcon.recipe.workflow.lib.path", new File(libPath).getParent());
- props.setProperty("falcon.recipe.src.cluster.hdfs.writeEndPoint", "jail://global:00");
+ props.setProperty("falcon.recipe.cluster.hdfs.writeEndPoint", "jail://global:00");
props.store(out, null);
out.close();
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication-template.xml b/webapp/src/test/resources/hdfs-replication-template.xml
new file mode 100644
index 0000000..b93cc0b
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-replication-template.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<process name="hdfs-replication" xmlns="uri:falcon:process:0.1">
+ <clusters>
+ <!-- source -->
+ <cluster name="##falcon.recipe.cluster.name##">
+ <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
+ </cluster>
+ </clusters>
+
+ <tags>_falcon_mirroring_type=HDFS</tags>
+
+ <parallel>1</parallel>
+ <!-- Dir replication needs to run only once to catch up -->
+ <order>LAST_ONLY</order>
+ <frequency>##falcon.recipe.frequency##</frequency>
+ <timezone>UTC</timezone>
+
+ <properties>
+ <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+ </properties>
+
+ <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
+ <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
+ <ACL/>
+</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties
new file mode 100644
index 0000000..1b485b4
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-replication.properties
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+falcon.recipe.cluster.hdfs.writeEndPoint=jail://global:00
+falcon.recipe.workflow.path=
+falcon.recipe.processName=
+falcon.recipe.cluster.name=
+falcon.recipe.cluster.validity.end=
+falcon.recipe.cluster.validity.start=2012-04-20T00:00Z
+falcon.recipe.workflow.name=hdfs-dr-workflow
+falcon.recipe.process.frequency=minutes(5)
+
+##### Retry policy properties
+
+falcon.recipe.retry.policy=periodic
+falcon.recipe.retry.delay=minutes(30)
+falcon.recipe.retry.attempts=3
+
+drSourceDir=/tmp/test1
+drSourceClusterFS=jail://global:00
+drTargetDir=/tmp/test1
+drTargetClusterFS=jail://global:00
+
+# Change it to specify the maximum number of mappers for DistCP
+distcpMaxMaps=1
+# Change it to specify the bandwidth in MB for each mapper in DistCP
+distcpMapBandwidth=100
+
+##### Email on failure
+drNotificationReceivers=NA
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/process.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/process.properties b/webapp/src/test/resources/process.properties
deleted file mode 100644
index 1eb282c..0000000
--- a/webapp/src/test/resources/process.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-falcon.recipe.src.cluster.hdfs.writeEndPoint=jail://global:00
-falcon.recipe.workflow.path=
-falcon.recipe.processName=
-falcon.recipe.src.cluster.name=
-falcon.recipe.inputFeedName=
-falcon.recipe.outputFeedName=
-