You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/07/20 00:06:07 UTC
falcon git commit: FALCON-2084 HCatReplicationTest are failing in
secure mode
Repository: falcon
Updated Branches:
refs/heads/master ec4a273a8 -> ec3e49e39
FALCON-2084 HCatReplicationTest are failing in secure mode
Removed the dependency on job-xml whose priority has changed because of some backward incompatible changes in Oozie - this patch will work with 4.2.0 and upcoming 4.3.0
Author: Venkat Ranganathan <ve...@hortonworks.com>
Reviewers: "Balu Vellanki <ba...@apache.org>, Ying Zheng <yz...@hortonworks.com>"
Closes #231 from vrangan/FALCON-2084
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ec3e49e3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ec3e49e3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ec3e49e3
Branch: refs/heads/master
Commit: ec3e49e393c45ca580795e47fb085c8e58a8f789
Parents: ec4a273
Author: Venkat Ranganathan <ve...@hortonworks.com>
Authored: Tue Jul 19 17:05:57 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Tue Jul 19 17:05:57 2016 -0700
----------------------------------------------------------------------
.../feed/FeedReplicationCoordinatorBuilder.java | 5 ---
.../feed/FeedReplicationWorkflowBuilder.java | 7 +---
.../feed/HCatReplicationWorkflowBuilder.java | 34 +++++++++++++++++++-
.../main/resources/action/feed/table-export.xml | 1 -
.../main/resources/action/feed/table-import.xml | 1 -
.../feed/OozieFeedWorkflowBuilderTest.java | 8 -----
6 files changed, 34 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
index b0e46f0..07d293c 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
@@ -275,11 +275,6 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F
Path scriptPath = new Path(buildPath, "scripts");
copyHiveScript(fs, scriptPath, IMPORT_HQL);
copyHiveScript(fs, scriptPath, EXPORT_HQL);
-
- // create hive conf to stagingDir
- Path confPath = new Path(buildPath + "/conf");
- persistHiveConfiguration(fs, confPath, srcCluster, "falcon-source-");
- persistHiveConfiguration(fs, confPath, trgCluster, "falcon-target-");
} catch (IOException e) {
throw new FalconException("Unable to create hive conf files", e);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
index 5a62130..010446b 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java
@@ -24,7 +24,6 @@ import org.apache.falcon.Tag;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.FeedHelper;
-import org.apache.falcon.entity.HiveUtil;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
@@ -79,11 +78,7 @@ public abstract class FeedReplicationWorkflowBuilder extends OozieOrchestrationW
marshal(cluster, workflow, buildPath);
Properties props = getProperties(buildPath, wfName);
props.putAll(createDefaultConfiguration(cluster));
- if (EntityUtil.isTableStorageType(cluster, entity)) {
- // todo: kludge send source hcat creds for coord dependency check to pass
- props.putAll(HiveUtil.getHiveCredentials(srcCluster));
- props.putAll(HiveUtil.getHiveCredentials(cluster));
- }
+
props.putAll(getWorkflowProperties(entity));
props.putAll(FeedHelper.getUserWorkflowProperties(getLifecycle()));
// Write out the config to config-default.xml
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
index 629485d..3da97d3 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java
@@ -22,15 +22,18 @@ import org.apache.falcon.FalconException;
import org.apache.falcon.Tag;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
+import org.apache.falcon.entity.HiveUtil;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.oozie.workflow.ACTION;
import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
+import org.apache.falcon.oozie.workflow.CONFIGURATION;
import org.apache.falcon.util.OozieUtils;
import org.apache.falcon.workflow.WorkflowExecutionArgs;
import javax.xml.bind.JAXBElement;
import java.util.Arrays;
+import java.util.Map;
import java.util.Properties;
/**
@@ -60,6 +63,15 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
//Add pre-processing
if (shouldPreProcess()) {
ACTION action = getPreProcessingAction(false, Tag.REPLICATION);
+ Properties hiveConf = HiveUtil.getHiveCredentials(src);
+ for (Map.Entry<Object, Object> e : hiveConf.entrySet()) {
+ CONFIGURATION.Property prop = new CONFIGURATION.Property();
+ prop.setName((String) e.getKey());
+ prop.setValue((String) e.getValue());
+ LOG.info("Adding config to replication hive preprocessing action : key = {} value = {}",
+ e.getKey(), e.getValue());
+ action.getJava().getConfiguration().getProperty().add(prop);
+ }
addHDFSServersConfig(action, src, target);
addTransition(action, EXPORT_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(action);
@@ -72,6 +84,16 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
OozieUtils.unMarshalHiveAction(export);
org.apache.falcon.oozie.hive.ACTION hiveExportAction = exportActionJaxbElement.getValue();
addHDFSServersConfig(hiveExportAction, src, target);
+ Properties hiveConf = HiveUtil.getHiveCredentials(src);
+ for (Map.Entry<Object, Object> e : hiveConf.entrySet()) {
+ org.apache.falcon.oozie.hive.CONFIGURATION.Property prop =
+ new org.apache.falcon.oozie.hive.CONFIGURATION.Property();
+ prop.setName((String) e.getKey());
+ prop.setValue((String) e.getValue());
+ LOG.info("Adding config to replication hive export action : key = {} value = {}",
+ e.getKey(), e.getValue());
+ hiveExportAction.getConfiguration().getProperty().add(prop);
+ }
OozieUtils.marshalHiveAction(export, exportActionJaxbElement);
addTransition(export, REPLICATION_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(export);
@@ -89,6 +111,16 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
OozieUtils.unMarshalHiveAction(importAction);
org.apache.falcon.oozie.hive.ACTION hiveImportAction = importActionJaxbElement.getValue();
addHDFSServersConfig(hiveImportAction, src, target);
+ Properties hiveConf2 = HiveUtil.getHiveCredentials(target);
+ for (Map.Entry<Object, Object> e : hiveConf2.entrySet()) {
+ org.apache.falcon.oozie.hive.CONFIGURATION.Property prop =
+ new org.apache.falcon.oozie.hive.CONFIGURATION.Property();
+ prop.setName((String) e.getKey());
+ prop.setValue((String) e.getValue());
+ LOG.info("Adding config to replication hive import action : key = {} value = {}",
+ e.getKey(), e.getValue());
+ hiveImportAction.getConfiguration().getProperty().add(prop);
+ }
OozieUtils.marshalHiveAction(importAction, importActionJaxbElement);
addTransition(importAction, CLEANUP_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
workflow.getDecisionOrForkOrJoin().add(importAction);
@@ -133,8 +165,8 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild
(org.apache.falcon.oozie.workflow.ACTION) object;
String actionName = action.getName();
if (PREPROCESS_ACTION_NAME.equals(actionName)) {
+
// add reference to hive-site conf to each action
- action.getJava().setJobXml("${wf:appPath()}/conf/falcon-source-hive-site.xml");
if (isSecurityEnabled) { // add a reference to credential in the action
action.setCred(SOURCE_HIVE_CREDENTIAL_NAME);
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/resources/action/feed/table-export.xml
----------------------------------------------------------------------
diff --git a/oozie/src/main/resources/action/feed/table-export.xml b/oozie/src/main/resources/action/feed/table-export.xml
index fcf1a1a..6bc214b 100644
--- a/oozie/src/main/resources/action/feed/table-export.xml
+++ b/oozie/src/main/resources/action/feed/table-export.xml
@@ -26,7 +26,6 @@
<prepare>
<delete path="${distcpSourcePaths}"/>
</prepare>
- <job-xml>${wf:appPath()}/conf/falcon-source-hive-site.xml</job-xml>
<configuration>
<property>
<name>mapred.job.queue.name</name>
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/resources/action/feed/table-import.xml
----------------------------------------------------------------------
diff --git a/oozie/src/main/resources/action/feed/table-import.xml b/oozie/src/main/resources/action/feed/table-import.xml
index 6e9a073..450e68a 100644
--- a/oozie/src/main/resources/action/feed/table-import.xml
+++ b/oozie/src/main/resources/action/feed/table-import.xml
@@ -20,7 +20,6 @@
<hive xmlns="uri:oozie:hive-action:0.2">
<job-tracker>${falconTargetJobTracker}</job-tracker>
<name-node>${falconTargetNameNode}</name-node>
- <job-xml>${wf:appPath()}/conf/falcon-target-hive-site.xml</job-xml>
<configuration>
<property>
<name>mapred.job.queue.name</name>
http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index b0fc230..2040748 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -561,9 +561,6 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
Assert.assertTrue(fs.exists(new Path(wfPath + "/scripts/falcon-table-export.hql")));
Assert.assertTrue(fs.exists(new Path(wfPath + "/scripts/falcon-table-import.hql")));
- Assert.assertTrue(fs.exists(new Path(wfPath + "/conf")));
- Assert.assertTrue(fs.exists(new Path(wfPath + "/conf/falcon-source-hive-site.xml")));
- Assert.assertTrue(fs.exists(new Path(wfPath + "/conf/falcon-target-hive-site.xml")));
HashMap<String, String> props = getCoordProperties(coord);
@@ -626,11 +623,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
private void assertReplicationHCatCredentials(WORKFLOWAPP wf, String wfPath) throws IOException {
FileSystem fs = trgMiniDFS.getFileSystem();
- Path hiveConfPath = new Path(wfPath, "conf/falcon-source-hive-site.xml");
- Assert.assertTrue(fs.exists(hiveConfPath));
- hiveConfPath = new Path(wfPath, "conf/falcon-target-hive-site.xml");
- Assert.assertTrue(fs.exists(hiveConfPath));
boolean isSecurityEnabled = SecurityUtil.isSecurityEnabled();
if (isSecurityEnabled) {
@@ -651,7 +644,6 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
}
if ("recordsize".equals(actionName)) {
- Assert.assertEquals(action.getJava().getJobXml(), "${wf:appPath()}/conf/falcon-source-hive-site.xml");
if (isSecurityEnabled) {
Assert.assertNotNull(action.getCred());
Assert.assertEquals(action.getCred(), "falconSourceHiveAuth");