You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by su...@apache.org on 2015/04/13 19:33:35 UTC
falcon git commit: FALCON-1129 In a secure cluster,
feed replication fails because of Authentication issues. Contributed
by Venkat Ranganathan
Repository: falcon
Updated Branches:
refs/heads/master f95581007 -> aeb6c86f4
FALCON-1129 In a secure cluster, feed replication fails because of Authentication issues. Contributed by Venkat Ranganathan
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/aeb6c86f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/aeb6c86f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/aeb6c86f
Branch: refs/heads/master
Commit: aeb6c86f431631fa02f1725a6352b0b802491d49
Parents: f955810
Author: Suhas Vasu <su...@inmobi.com>
Authored: Mon Apr 13 23:01:50 2015 +0530
Committer: Suhas Vasu <su...@inmobi.com>
Committed: Mon Apr 13 23:03:07 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 6 ++++++
.../oozie/feed/FSReplicationWorkflowBuilder.java | 4 ++++
.../oozie/feed/FeedReplicationWorkflowBuilder.java | 15 ++++++++++++++-
.../oozie/feed/HCatReplicationWorkflowBuilder.java | 14 ++++----------
4 files changed, 28 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/aeb6c86f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bde0720..3028c27 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-1129 In a secure cluster, feed replication fails because of
+ Authentication issues (Venkat Ranganathan via Suhas Vasu)
+
FALCON-1141 Reverse Lookup for feed in prism fails with BadRequest
(Ajay Yadava via Suhas Vasu)
@@ -155,6 +158,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1)
(Suhas vasu)
BUG FIXES
+ FALCON-1129 In a secure cluster, feed replication fails because of
+ Authentication issues (Venkat Ranganathan via Suhas Vasu)
+
FALCON-1141 Reverse Lookup for feed in prism fails with BadRequest
(Ajay Yadava via Suhas Vasu)
http://git-wip-us.apache.org/repos/asf/falcon/blob/aeb6c86f/oozie/src/main/java/org/apache/falcon/oozie/feed/FSReplicationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FSReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FSReplicationWorkflowBuilder.java
index 6feb32e..1d97204 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FSReplicationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FSReplicationWorkflowBuilder.java
@@ -45,6 +45,7 @@ public class FSReplicationWorkflowBuilder extends FeedReplicationWorkflowBuilder
//Add pre-processing
if (shouldPreProcess()) {
ACTION action = getPreProcessingAction(false, Tag.REPLICATION);
+ addHDFSServersConfig(action, src, target);
addTransition(action, REPLICATION_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(action);
start = PREPROCESS_ACTION_NAME;
@@ -52,15 +53,18 @@ public class FSReplicationWorkflowBuilder extends FeedReplicationWorkflowBuilder
//Add replication
ACTION replication = unmarshalAction(REPLICATION_ACTION_TEMPLATE);
+ addHDFSServersConfig(replication, src, target);
addTransition(replication, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(replication);
//Add post-processing actions
ACTION success = getSuccessPostProcessAction();
+ addHDFSServersConfig(success, src, target);
addTransition(success, OK_ACTION_NAME, FAIL_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(success);
ACTION fail = getFailPostProcessAction();
+ addHDFSServersConfig(fail, src, target);
addTransition(fail, FAIL_ACTION_NAME, FAIL_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(fail);
http://git-wip-us.apache.org/repos/asf/falcon/blob/aeb6c86f/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
index 288e9de..aa936ad 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
@@ -21,12 +21,15 @@ package org.apache.falcon.oozie.feed;
import org.apache.falcon.FalconException;
import org.apache.falcon.LifeCycle;
import org.apache.falcon.Tag;
+import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.oozie.OozieOrchestrationWorkflowBuilder;
+import org.apache.falcon.oozie.workflow.ACTION;
+import org.apache.falcon.oozie.workflow.CONFIGURATION;
import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
import org.apache.hadoop.fs.Path;
@@ -55,6 +58,16 @@ public abstract class FeedReplicationWorkflowBuilder extends OozieOrchestrationW
marshal(cluster, workflow, buildPath);
return getProperties(buildPath, wfName);
}
-
+ protected ACTION addHDFSServersConfig(ACTION action, Cluster sourceCluster, Cluster targetCluster) {
+ if (isSecurityEnabled) {
+ // this is to ensure that the delegation tokens are checked out for both clusters
+ CONFIGURATION.Property property = new CONFIGURATION.Property();
+ property.setName("mapreduce.job.hdfs-servers");
+ property.setValue(ClusterHelper.getReadOnlyStorageUrl(sourceCluster)
+ + "," + ClusterHelper.getStorageUrl(targetCluster));
+ action.getJava().getConfiguration().getProperty().add(property);
+ }
+ return action;
+ }
protected abstract WORKFLOWAPP getWorkflow(Cluster src, Cluster target) throws FalconException;
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/aeb6c86f/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
index 30ca0a8..72bbca4 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
@@ -20,12 +20,10 @@ package org.apache.falcon.oozie.feed;
import org.apache.falcon.FalconException;
import org.apache.falcon.Tag;
-import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.oozie.workflow.ACTION;
-import org.apache.falcon.oozie.workflow.CONFIGURATION;
import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
import java.util.Arrays;
@@ -57,6 +55,7 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
//Add pre-processing
if (shouldPreProcess()) {
ACTION action = getPreProcessingAction(false, Tag.REPLICATION);
+ addHDFSServersConfig(action, src, target);
addTransition(action, EXPORT_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(action);
start = PREPROCESS_ACTION_NAME;
@@ -84,10 +83,12 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
//Add post-processing actions
ACTION success = getSuccessPostProcessAction();
+ addHDFSServersConfig(success, src, target);
addTransition(success, OK_ACTION_NAME, FAIL_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(success);
ACTION fail = getFailPostProcessAction();
+ addHDFSServersConfig(fail, src, target);
addTransition(fail, FAIL_ACTION_NAME, FAIL_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(fail);
@@ -130,14 +131,7 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
action.setCred(TARGET_HIVE_CREDENTIAL_NAME);
}
} else if (REPLICATION_ACTION_NAME.equals(actionName)) {
- if (isSecurityEnabled) {
- // this is to ensure that the delegation tokens are checked out for both clusters
- CONFIGURATION.Property property = new CONFIGURATION.Property();
- property.setName("mapreduce.job.hdfs-servers");
- property.setValue(ClusterHelper.getReadOnlyStorageUrl(sourceCluster)
- + "," + ClusterHelper.getStorageUrl(targetCluster));
- action.getJava().getConfiguration().getProperty().add(property);
- }
+ addHDFSServersConfig(action, sourceCluster, targetCluster);
}
}
}