You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/11/24 13:43:56 UTC
falcon git commit: FALCON-1588 Add ability to provide the path for
recipe files in command line. Contributed by Peeyush Bishnoi.
Repository: falcon
Updated Branches:
refs/heads/master ef4e8a4b1 -> 6676032c3
FALCON-1588 Add ability to provide the path for recipe files in command line. Contributed by Peeyush Bishnoi.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6676032c
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6676032c
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6676032c
Branch: refs/heads/master
Commit: 6676032c3e2746f965a25d02a17bd107b3074df4
Parents: ef4e8a4
Author: Ajay Yadava <aj...@gmail.com>
Authored: Tue Nov 24 16:08:40 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Tue Nov 24 16:08:40 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/falcon/cli/FalconRecipeCLI.java | 24 ++++++++++++++++-
.../org/apache/falcon/client/FalconClient.java | 28 +++++++++++++++-----
src/conf/client.properties | 1 +
4 files changed, 48 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 62e5725..13d3439 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-1588 Add ability to provide the path for recipe files in command line(Peeyush Bishnoi via Ajay Yadava)
+
FALCON-1573 Supply user-defined properties to Oozie workflows during schedule(Daniel Del Castillo via Ajay Yadava)
FALCON-1559 Config changes required for native scheduler (Pallavi Rao)
http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java
index b93fed2..82053f9 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java
@@ -21,6 +21,7 @@ package org.apache.falcon.cli;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.client.FalconCLIException;
import org.apache.falcon.client.FalconClient;
@@ -39,6 +40,7 @@ public class FalconRecipeCLI extends FalconCLI {
private static final String RECIPE_NAME = "name";
private static final String RECIPE_OPERATION= "operation";
private static final String RECIPE_TOOL_CLASS_NAME = "tool";
+ private static final String RECIPE_PROPERTIES_FILE = "properties";
public Options createRecipeOptions() {
Options recipeOptions = new Options();
@@ -54,6 +56,9 @@ public class FalconRecipeCLI extends FalconCLI {
Option recipeOperation = new Option(RECIPE_OPERATION, true, "recipe operation");
recipeOptions.addOption(recipeOperation);
+ Option recipeProperties = new Option(RECIPE_PROPERTIES_FILE, true, "recipe properties file path");
+ recipeOptions.addOption(recipeProperties);
+
Option skipDryRunOperation = new Option(SKIPDRYRUN_OPT, false, "skip dryrun operation");
recipeOptions.addOption(skipDryRunOperation);
@@ -72,18 +77,20 @@ public class FalconRecipeCLI extends FalconCLI {
String recipeName = commandLine.getOptionValue(RECIPE_NAME);
String recipeToolClass = commandLine.getOptionValue(RECIPE_TOOL_CLASS_NAME);
String recipeOperation = commandLine.getOptionValue(RECIPE_OPERATION);
+ String recipePropertiesFile = commandLine.getOptionValue(RECIPE_PROPERTIES_FILE);
String doAsUser = commandLine.getOptionValue(DO_AS_OPT);
validateNotEmpty(recipeName, RECIPE_NAME);
validateNotEmpty(recipeOperation, RECIPE_OPERATION);
validateRecipeOperations(recipeOperation);
+ validateRecipePropertiesFile(recipePropertiesFile, recipeName);
Boolean skipDryRun = null;
if (optionsList.contains(SKIPDRYRUN_OPT)) {
skipDryRun = true;
}
String result = client.submitRecipe(recipeName, recipeToolClass,
- recipeOperation, skipDryRun, doAsUser).toString();
+ recipeOperation, recipePropertiesFile, skipDryRun, doAsUser).toString();
OUT.get().println(result);
}
@@ -96,4 +103,19 @@ public class FalconRecipeCLI extends FalconCLI {
throw new FalconCLIException("Allowed Recipe operations: "
+ java.util.Arrays.asList((RecipeOperation.values())));
}
+
+ private static void validateRecipePropertiesFile(String recipePropertiesFile, String recipeName)
+ throws FalconCLIException {
+ if (StringUtils.isBlank(recipePropertiesFile)) {
+ return;
+ }
+
+ String []fileSplits = recipePropertiesFile.split("/");
+ String recipePropertiesName = (fileSplits[fileSplits.length-1]).split("\\.")[0];
+ if (recipePropertiesName.equals(recipeName)) {
+ return;
+ }
+
+ throw new FalconCLIException("Provided properties file name do match with recipe name: " +recipeName);
+ }
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 471cab4..b4b1762 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -1102,21 +1102,37 @@ public class FalconClient extends AbstractFalconClient {
return sendMetadataLineageRequest(MetadataOperations.EDGES, id, doAsUser);
}
+ private String getRecipePath(String recipePropertiesFile) throws FalconCLIException {
+ String recipePath = null;
+ if (StringUtils.isNotBlank(recipePropertiesFile)) {
+ File file = new File(recipePropertiesFile);
+ if (file.exists()) {
+ recipePath = file.getAbsoluteFile().getParentFile().getAbsolutePath();
+ }
+ } else {
+ recipePath = clientProperties.getProperty("falcon.recipe.path");
+ }
+
+ return recipePath;
+ }
+
public APIResult submitRecipe(String recipeName,
String recipeToolClassName,
final String recipeOperation,
+ String recipePropertiesFile,
Boolean skipDryRun,
final String doAsUser) throws FalconCLIException {
- String recipePath = clientProperties.getProperty("falcon.recipe.path");
+ String recipePath = getRecipePath(recipePropertiesFile);
if (StringUtils.isEmpty(recipePath)) {
- throw new FalconCLIException("falcon.recipe.path is not set in client.properties");
+ throw new FalconCLIException("falcon.recipe.path is not set in client.properties or properties "
+ + " file is not provided");
}
- String recipeFilePath = recipePath + File.separator + recipeName + TEMPLATE_SUFFIX;
- File file = new File(recipeFilePath);
+ String templateFilePath = recipePath + File.separator + recipeName + TEMPLATE_SUFFIX;
+ File file = new File(templateFilePath);
if (!file.exists()) {
- throw new FalconCLIException("Recipe template file does not exist : " + recipeFilePath);
+ throw new FalconCLIException("Recipe template file does not exist : " + templateFilePath);
}
String propertiesFilePath = recipePath + File.separator + recipeName + PROPERTIES_SUFFIX;
@@ -1139,7 +1155,7 @@ public class FalconClient extends AbstractFalconClient {
processFile = f.getAbsolutePath();
String[] args = {
- "-" + RecipeToolArgs.RECIPE_FILE_ARG.getName(), recipeFilePath,
+ "-" + RecipeToolArgs.RECIPE_FILE_ARG.getName(), templateFilePath,
"-" + RecipeToolArgs.RECIPE_PROPERTIES_FILE_ARG.getName(), propertiesFilePath,
"-" + RecipeToolArgs.RECIPE_PROCESS_XML_FILE_PATH_ARG.getName(), processFile,
"-" + RecipeToolArgs.RECIPE_OPERATION_ARG.getName(), recipeOperation,
http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/src/conf/client.properties
----------------------------------------------------------------------
diff --git a/src/conf/client.properties b/src/conf/client.properties
index 7925009..900e043 100644
--- a/src/conf/client.properties
+++ b/src/conf/client.properties
@@ -22,3 +22,4 @@
#########################################################################
falcon.url=https://localhost:15443/
+#falcon.recipe.path=