You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/03/28 08:43:34 UTC
falcon git commit: FALCON-1874 Import and Export fails with HDFS as
src/dest
Repository: falcon
Updated Branches:
refs/heads/master 8e79ba81a -> 10f3843ad
FALCON-1874 Import and Export fails with HDFS as src/dest
1. job-xml was pointing to hive-site.xml even for HDFS, cause workflow to fail.
2. datain element was repeated in export coordinator, causing workflow to fail.
Author: Pallavi Rao <pa...@inmobi.com>
Reviewers: @vramachan, @PraveenAdlakha
Closes #78 from pallavi-rao/FALCON-1874
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/10f3843a
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/10f3843a
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/10f3843a
Branch: refs/heads/master
Commit: 10f3843addd509e3f77bad9737ee2ddb33681a94
Parents: 8e79ba8
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Mon Mar 28 12:13:32 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Mon Mar 28 12:13:32 2016 +0530
----------------------------------------------------------------------
.../org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java | 3 +--
.../org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java | 3 +--
.../org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java | 1 -
.../main/java/org/apache/falcon/oozie/ImportExportCommon.java | 5 ++++-
4 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
index 284c4a3..e7ca869 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
@@ -64,8 +64,7 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder {
org.apache.falcon.oozie.sqoop.ACTION sqoopExport = actionJaxbElement.getValue();
Properties props = new Properties();
- ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath);
- sqoopExport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
+ ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath, sqoopExport);
OozieUtils.marshalSqoopAction(action, actionJaxbElement);
addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
index 3e24428..1efa368 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
@@ -66,8 +66,7 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder {
org.apache.falcon.oozie.sqoop.ACTION sqoopImport = actionJaxbElement.getValue();
Properties props = new Properties();
- ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath);
- sqoopImport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
+ ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath, sqoopImport);
OozieUtils.marshalSqoopAction(action, actionJaxbElement);
addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);
http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
index 4437d8b..e37accd 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
@@ -125,7 +125,6 @@ public class FeedExportCoordinatorBuilder extends OozieCoordinatorBuilder<Feed>
datain.setDataset(EXPORT_DATASET_NAME);
org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
- datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
return datain;
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
index 52c7820..fafd56e 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
@@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.datasource.Credential;
import org.apache.falcon.entity.v0.datasource.Credentialtype;
import org.apache.falcon.entity.v0.datasource.Datasource;
import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.oozie.sqoop.ACTION;
import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
import org.apache.falcon.security.SecurityUtil;
import org.apache.hadoop.fs.Path;
@@ -88,7 +89,8 @@ public final class ImportExportCommon {
}
public static void addHCatalogProperties(Properties props, Feed entity, Cluster cluster,
- WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder<Feed> wBuilder, Path buildPath)
+ WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder<Feed> wBuilder,
+ Path buildPath, ACTION sqoopAction)
throws FalconException {
if (FeedHelper.getStorageType(entity, cluster) == Storage.TYPE.TABLE) {
wBuilder.createHiveConfiguration(cluster, buildPath, "");
@@ -98,6 +100,7 @@ public final class ImportExportCommon {
wBuilder.addHCatalogCredentials(workflow, cluster,
OozieOrchestrationWorkflowBuilder.HIVE_CREDENTIAL_NAME, FALCON_IMPORT_SQOOP_ACTIONS);
}
+ sqoopAction.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
}
}
private static void addHCatalogShareLibs(Properties props) throws FalconException {